From 9133d0464f8a6bff0f72e2966279c5006a084d29 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 12 May 2026 02:23:43 -0700 Subject: [PATCH 1/6] feat: pre-cast constants (#7926) * init impl Signed-off-by: Ruihang Xia * handle no cast Signed-off-by: Ruihang Xia * refactor using common-expr Signed-off-by: Ruihang Xia * extend matching pattern Signed-off-by: Ruihang Xia * more tests Signed-off-by: Ruihang Xia * simplification Signed-off-by: Ruihang Xia * fix zero timestamp Signed-off-by: Ruihang Xia * fix: normalize sqlness partition count output Signed-off-by: Ruihang Xia * fix: normalize remaining sqlness plan output Signed-off-by: Ruihang Xia * fix: normalize sqlness repartition details in tql explain Signed-off-by: Ruihang Xia * fix: tighten const normalization casts * test: normalize standalone tql explain repartition output * resolve cr comments Signed-off-by: Ruihang Xia * simplify Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- Cargo.lock | 1 + Cargo.toml | 2 + src/query/Cargo.toml | 1 + src/query/src/optimizer.rs | 1 + .../src/optimizer/const_normalization.rs | 1133 +++++++++++++++++ src/query/src/query_engine/state.rs | 2 + .../common/tql-explain-analyze/explain.result | 392 ++++++ .../common/tql-explain-analyze/explain.sql | 23 + .../time_index_filter_pushdown.result | 115 +- .../optimizer/time_index_filter_pushdown.sql | 72 +- 10 files changed, 1721 insertions(+), 21 deletions(-) create mode 100644 src/query/src/optimizer/const_normalization.rs diff --git a/Cargo.lock b/Cargo.lock index 4d35a7e9bb..e4918c386c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11088,6 +11088,7 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-optimizer", "datafusion-physical-expr", diff --git a/Cargo.toml b/Cargo.toml index 969cd7da38..c9ddba912a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ datafusion = "=52.1" datafusion-common = "=52.1" datafusion-datasource = "=52.1" datafusion-expr = "=52.1" +datafusion-expr-common = "=52.1" datafusion-functions = "=52.1" datafusion-functions-aggregate-common = "=52.1" datafusion-functions-window-common = "=52.1" @@ -340,6 +341,7 @@ rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" } diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml index 6bbd9e1dd1..b92921d29b 100644 --- a/src/query/Cargo.toml +++ b/src/query/Cargo.toml @@ -40,6 +40,7 @@ common-time.workspace = true datafusion.workspace = true datafusion-common.workspace = true datafusion-expr.workspace = true +datafusion-expr-common.workspace = true datafusion-functions.workspace = true datafusion-optimizer.workspace = true datafusion-physical-expr.workspace = true diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs index b1945f2020..b85320a495 100644 --- a/src/query/src/optimizer.rs +++ b/src/query/src/optimizer.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod const_normalization; pub mod constant_term; pub mod count_nest_aggr; pub mod count_wildcard; diff --git a/src/query/src/optimizer/const_normalization.rs b/src/query/src/optimizer/const_normalization.rs new file mode 100644 index 0000000000..fac0e90538 --- /dev/null +++ b/src/query/src/optimizer/const_normalization.rs @@ -0,0 +1,1133 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit}; +use datafusion::config::ConfigOptions; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter}; +use datafusion_common::{DFSchemaRef, Result, ScalarValue}; +use datafusion_expr::expr::{Cast, InList, Like, TryCast}; +use datafusion_expr::{Between, BinaryExpr, Expr, ExprSchemable, LogicalPlan, Operator, lit}; +use datafusion_expr_common::casts::try_cast_literal_to_type; +use datafusion_optimizer::analyzer::AnalyzerRule; + +use crate::plan::ExtractExpr; + +/// ConstNormalizationRule rewrites castable constants against their +/// non-constant comparison operand ahead of filter pushdown. +#[derive(Debug)] +pub struct ConstNormalizationRule; + +impl AnalyzerRule for ConstNormalizationRule { + fn analyze(&self, plan: LogicalPlan, _config: &ConfigOptions) -> Result { + plan.transform(|plan| match plan { + LogicalPlan::Filter(filter) => { + let schema = filter.input.schema().clone(); + rewrite_plan_exprs(LogicalPlan::Filter(filter), schema) + } + LogicalPlan::TableScan(scan) => { + let schema = scan.projected_schema.clone(); + rewrite_plan_exprs(LogicalPlan::TableScan(scan), schema) + } + _ => Ok(Transformed::no(plan)), + }) + .map(|x| x.data) + } + + fn name(&self) -> &str { + "ConstNormalizationRule" + } +} + +fn rewrite_plan_exprs(plan: LogicalPlan, schema: DFSchemaRef) -> Result> { + let mut rewriter = ConstNormalizationRewriter { + schema, + transformed: false, + }; + let exprs = plan + .expressions_consider_join() + .into_iter() + .map(|expr| expr.rewrite(&mut rewriter).map(|rewritten| rewritten.data)) + .collect::>>()?; + if !rewriter.transformed { + return Ok(Transformed::no(plan)); + } + + let inputs = plan.inputs().into_iter().cloned().collect::>(); + plan.with_new_exprs(exprs, inputs).map(Transformed::yes) +} + +struct ConstNormalizationRewriter { + schema: DFSchemaRef, + transformed: bool, +} + +impl TreeNodeRewriter for ConstNormalizationRewriter { + type Node = Expr; + + fn f_down(&mut self, expr: Expr) -> Result> { + let recursion = if matches!( + expr, + Expr::Exists(_) | Expr::InSubquery(_) | Expr::ScalarSubquery(_) + ) { + TreeNodeRecursion::Jump + } else { + TreeNodeRecursion::Continue + }; + + Ok(Transformed::new(expr, false, recursion)) + } + + fn f_up(&mut self, expr: Expr) -> Result> { + let rewritten = rewrite_expr_node(expr, &self.schema)?; + self.transformed |= rewritten.transformed; + Ok(rewritten) + } +} + +fn rewrite_expr_node(expr: Expr, schema: &DFSchemaRef) -> Result> { + match expr { + Expr::BinaryExpr(binary) => match rewrite_binary_expr(binary.clone(), schema)? { + Some(expr) => Ok(Transformed::yes(expr)), + None => Ok(Transformed::no(Expr::BinaryExpr(binary))), + }, + Expr::Between(between) => match rewrite_between_expr(between.clone(), schema)? { + Some(expr) => Ok(Transformed::yes(expr)), + None => Ok(Transformed::no(Expr::Between(between))), + }, + Expr::InList(in_list) => match rewrite_in_list_expr(in_list.clone(), schema)? { + Some(expr) => Ok(Transformed::yes(expr)), + None => Ok(Transformed::no(Expr::InList(in_list))), + }, + Expr::Like(like) => rewrite_like_expr(like, PatternMatchKind::Like, schema), + Expr::SimilarTo(like) => rewrite_like_expr(like, PatternMatchKind::SimilarTo, schema), + expr => Ok(Transformed::no(expr)), + } +} + +fn rewrite_between_expr(between: Between, schema: &DFSchemaRef) -> Result> { + let Between { + expr, + negated, + low, + high, + } = between; + let expr = *expr; + let low_expr = *low; + let high_expr = *high; + let Some((target, constants)) = + extract_rewrite_operands(&expr, &[low_expr.clone(), high_expr.clone()], schema)? + else { + return Ok(None); + }; + + if let Some(mut constants) = target.normalize_constants(&constants) { + let high = constants + .pop() + .expect("between normalization expects high constant"); + let low = constants + .pop() + .expect("between normalization expects low constant"); + return Ok(Some(Expr::Between(Between { + expr: Box::new(target.expr.clone()), + negated, + low: Box::new(lit(low)), + high: Box::new(lit(high)), + }))); + } + + Ok((!negated) + .then(|| target.normalize_timestamp_between(&constants[0], &constants[1])) + .flatten()) +} + +fn rewrite_in_list_expr(in_list: InList, schema: &DFSchemaRef) -> Result> { + let InList { + expr, + list, + negated, + } = in_list; + let expr = *expr; + let Some((target, constants)) = extract_rewrite_operands(&expr, &list, schema)? else { + return Ok(None); + }; + + Ok(target.normalize_constants(&constants).map(|constants| { + target + .expr + .clone() + .in_list(constants.into_iter().map(lit).collect(), negated) + })) +} + +fn rewrite_like_expr( + like: Like, + kind: PatternMatchKind, + schema: &DFSchemaRef, +) -> Result> { + let original = match kind { + PatternMatchKind::Like => Expr::Like(like.clone()), + PatternMatchKind::SimilarTo => Expr::SimilarTo(like.clone()), + }; + let Like { + negated, + expr, + pattern, + escape_char, + case_insensitive, + } = like; + let expr = *expr; + let pattern = *pattern; + let Some((target, constants)) = + extract_rewrite_operands(&expr, std::slice::from_ref(&pattern), schema)? + else { + return Ok(Transformed::no(original)); + }; + let Some(mut constants) = target.normalize_constants(&constants) else { + return Ok(Transformed::no(original)); + }; + + let pattern = lit(constants + .pop() + .expect("pattern normalization expects one constant")); + let like = Like::new( + negated, + Box::new(target.expr.clone()), + Box::new(pattern), + escape_char, + case_insensitive, + ); + let rewritten = match kind { + PatternMatchKind::Like => Expr::Like(like), + PatternMatchKind::SimilarTo => Expr::SimilarTo(like), + }; + Ok(Transformed::yes(rewritten)) +} + +fn rewrite_binary_expr(binary: BinaryExpr, schema: &DFSchemaRef) -> Result> { + if !binary.op.supports_propagation() { + return Ok(None); + } + + let BinaryExpr { left, op, right } = binary; + let left = *left; + let right = *right; + if let Some(expr) = rewrite_binary_side(left.clone(), op, right.clone(), schema)? { + return Ok(Some(expr)); + } + + let Some(swapped_op) = op.swap() else { + return Ok(None); + }; + + rewrite_binary_side(right, swapped_op, left, schema) +} + +fn rewrite_binary_side( + target_expr: Expr, + op: Operator, + constant_expr: Expr, + schema: &DFSchemaRef, +) -> Result> { + let Some((target, constants)) = + extract_rewrite_operands(&target_expr, std::slice::from_ref(&constant_expr), schema)? + else { + return Ok(None); + }; + + if let Some(mut constants) = target.normalize_constants(&constants) { + let constant = constants + .pop() + .expect("binary normalization expects one constant"); + return Ok(Some(Expr::BinaryExpr(BinaryExpr { + left: Box::new(target.expr.clone()), + op, + right: Box::new(lit(constant)), + }))); + } + + Ok(target.normalize_timestamp_binary(op, &constants[0])) +} + +fn extract_rewrite_operands( + target_expr: &Expr, + constant_exprs: &[Expr], + schema: &DFSchemaRef, +) -> Result)>> { + let Some(target) = extract_normalization_target(target_expr, schema)? else { + return Ok(None); + }; + + extract_constant_scalars(constant_exprs) + .map(|constants| constants.map(|constants| (target, constants))) +} + +#[derive(Clone)] +struct NormalizationTarget { + expr: Expr, + data_type: DataType, + kind: NormalizationKind, +} + +#[derive(Clone)] +enum NormalizationKind { + /// The cast preserves every source value exactly, so literals can be cast directly. + Lossless, + /// The cast drops timestamp precision and must widen predicate bounds to preserve semantics. + TimestampDowncast { + source_unit: ArrowTimeUnit, + target_unit: ArrowTimeUnit, + timezone: Option>, + }, +} + +impl NormalizationTarget { + /// Normalizes constants for rewrites that can preserve the original predicate with a direct + /// literal cast. Timestamp precision-changing casts are handled by timestamp-specific helpers. + fn normalize_constants(&self, constants: &[ScalarValue]) -> Option> { + constants + .iter() + .map(|constant| self.normalize_constant(constant)) + .collect() + } + + fn normalize_constant(&self, constant: &ScalarValue) -> Option { + match self.kind { + NormalizationKind::TimestampDowncast { .. } => None, + NormalizationKind::Lossless => try_cast_literal_to_type(constant, &self.data_type), + } + } + + /// Rewrites predicates over timestamp downcasts into source-side half-open bounds. + fn normalize_timestamp_binary(&self, op: Operator, constant: &ScalarValue) -> Option { + let NormalizationKind::TimestampDowncast { + source_unit, + target_unit, + timezone, + } = &self.kind + else { + return None; + }; + + let constant = constant + .cast_to(&DataType::Timestamp(*target_unit, timezone.clone())) + .ok()?; + let value = timestamp_scalar_value(&constant)?; + let bound = match op { + Operator::GtEq => lower_bound_for_ge(value, *source_unit, *target_unit)?, + Operator::Gt => lower_bound_for_ge(value.checked_add(1)?, *source_unit, *target_unit)?, + Operator::Lt => lower_bound_for_ge(value, *source_unit, *target_unit)?, + Operator::LtEq => { + lower_bound_for_ge(value.checked_add(1)?, *source_unit, *target_unit)? + } + _ => return None, + }; + + let normalized_op = match op { + Operator::GtEq | Operator::Gt => Operator::GtEq, + Operator::Lt | Operator::LtEq => Operator::Lt, + _ => return None, + }; + + Some(match normalized_op { + Operator::GtEq => self.expr.clone().gt_eq(lit(timestamp_scalar( + *source_unit, + timezone.clone(), + bound, + ))), + Operator::Lt => { + self.expr + .clone() + .lt(lit(timestamp_scalar(*source_unit, timezone.clone(), bound))) + } + _ => unreachable!("timestamp normalization only rewrites to >= or <"), + }) + } + + /// Rewrites `BETWEEN` over timestamp downcasts into an inclusive lower bound and exclusive + /// upper bound over the source timestamp unit. + fn normalize_timestamp_between(&self, low: &ScalarValue, high: &ScalarValue) -> Option { + let NormalizationKind::TimestampDowncast { + source_unit, + target_unit, + timezone, + } = &self.kind + else { + return None; + }; + + let target_type = DataType::Timestamp(*target_unit, timezone.clone()); + let low = low.cast_to(&target_type).ok()?; + let high = high.cast_to(&target_type).ok()?; + let low = timestamp_scalar_value(&low)?; + let high = timestamp_scalar_value(&high)?; + + let lower = lower_bound_for_ge(low, *source_unit, *target_unit)?; + let upper = lower_bound_for_ge(high.checked_add(1)?, *source_unit, *target_unit)?; + + Some( + self.expr + .clone() + .gt_eq(lit(timestamp_scalar(*source_unit, timezone.clone(), lower))) + .and(self.expr.clone().lt(lit(timestamp_scalar( + *source_unit, + timezone.clone(), + upper, + )))), + ) + } +} + +/// Returns the non-constant side we should normalize against. +/// +/// Plain expressions normalize literals to their own type. Cast expressions only participate when +/// the cast is lossless or when timestamp downcasts can be rewritten as wider source-side bounds. +fn extract_normalization_target( + expr: &Expr, + schema: &DFSchemaRef, +) -> Result> { + if extract_constant_scalar(expr)?.is_some() { + return Ok(None); + } + + let Some((_, source_expr, target_type)) = extract_cast_input(expr) else { + return Ok(Some(NormalizationTarget { + expr: expr.clone(), + data_type: expr.get_type(schema)?, + kind: NormalizationKind::Lossless, + })); + }; + + let data_type = source_expr.get_type(schema)?; + let Some(kind) = classify_normalization_kind(&data_type, target_type) else { + return Ok(None); + }; + + Ok(Some(NormalizationTarget { + expr: source_expr.clone(), + data_type, + kind, + })) +} + +fn classify_normalization_kind( + source_type: &DataType, + target_type: &DataType, +) -> Option { + // Timestamp casts that change precision need boundary-aware rewrites. A finer target literal + // may not map exactly back to the coarser source unit, so the generic lossless path is only + // safe for timestamp casts that keep the same unit. + if is_lossless_cast(source_type, target_type) { + return Some(NormalizationKind::Lossless); + } + + match (source_type, target_type) { + ( + DataType::Timestamp(source_unit, source_tz), + DataType::Timestamp(target_unit, target_tz), + ) if source_tz == target_tz + && time_unit_rank(*source_unit) > time_unit_rank(*target_unit) => + { + Some(NormalizationKind::TimestampDowncast { + source_unit: *source_unit, + target_unit: *target_unit, + timezone: source_tz.clone(), + }) + } + _ => None, + } +} + +/// Returns whether every value of `source_type` is representable in `target_type`. +fn is_lossless_cast(source_type: &DataType, target_type: &DataType) -> bool { + match (source_type, target_type) { + (DataType::Int8, DataType::Int16 | DataType::Int32 | DataType::Int64) + | (DataType::Int16, DataType::Int32 | DataType::Int64) + | (DataType::Int32, DataType::Int64) + | (DataType::UInt8, DataType::UInt16 | DataType::UInt32 | DataType::UInt64) + | (DataType::UInt8, DataType::Int16 | DataType::Int32 | DataType::Int64) + | (DataType::UInt16, DataType::UInt32 | DataType::UInt64) + | (DataType::UInt16, DataType::Int32 | DataType::Int64) + | (DataType::UInt32, DataType::UInt64 | DataType::Int64) + | (DataType::Utf8, DataType::Utf8View | DataType::LargeUtf8) => true, + ( + DataType::Timestamp(source_unit, source_tz), + DataType::Timestamp(target_unit, target_tz), + ) => source_tz == target_tz && source_unit == target_unit, + _ => false, + } +} + +#[derive(Clone, Copy)] +enum PatternMatchKind { + Like, + SimilarTo, +} + +fn extract_constant_scalars(exprs: &[Expr]) -> Result>> { + let mut values = Vec::with_capacity(exprs.len()); + for expr in exprs { + let Some(value) = extract_constant_scalar(expr)? else { + return Ok(None); + }; + values.push(value); + } + + Ok(Some(values)) +} + +/// Extracts a literal scalar from an expression, folding constant `CAST` and `TRY_CAST` nodes. +fn extract_constant_scalar(expr: &Expr) -> Result> { + if let Some(value) = expr.as_literal() { + return Ok(Some(value.clone())); + } + + let Some((kind, expr, data_type)) = extract_cast_input(expr) else { + return Ok(None); + }; + + match kind { + CastInputKind::Cast => extract_constant_scalar(expr)? + .map(|value| value.cast_to(data_type)) + .transpose(), + CastInputKind::TryCast => { + Ok(extract_constant_scalar(expr)?.and_then(|value| value.cast_to(data_type).ok())) + } + } +} + +#[derive(Clone, Copy)] +enum CastInputKind { + Cast, + TryCast, +} + +/// Returns the input expression and target type for `CAST` and `TRY_CAST` expressions. +fn extract_cast_input(expr: &Expr) -> Option<(CastInputKind, &Expr, &DataType)> { + match expr { + Expr::Cast(Cast { expr, data_type }) => { + Some((CastInputKind::Cast, expr.as_ref(), data_type)) + } + Expr::TryCast(TryCast { expr, data_type }) => { + Some((CastInputKind::TryCast, expr.as_ref(), data_type)) + } + _ => None, + } +} + +fn time_unit_rank(unit: ArrowTimeUnit) -> usize { + match unit { + ArrowTimeUnit::Second => 0, + ArrowTimeUnit::Millisecond => 1, + ArrowTimeUnit::Microsecond => 2, + ArrowTimeUnit::Nanosecond => 3, + } +} + +fn time_unit_scale(unit: ArrowTimeUnit) -> i64 { + match unit { + ArrowTimeUnit::Second => 1, + ArrowTimeUnit::Millisecond => 1_000, + ArrowTimeUnit::Microsecond => 1_000_000, + ArrowTimeUnit::Nanosecond => 1_000_000_000, + } +} + +/// Returns the number of source-unit ticks in one target-unit tick for finer-to-coarser casts. +fn finer_to_coarser_ratio(source_unit: ArrowTimeUnit, target_unit: ArrowTimeUnit) -> Option { + let source_scale = time_unit_scale(source_unit); + let target_scale = time_unit_scale(target_unit); + (source_scale >= target_scale).then_some(source_scale / target_scale) +} + +/// Returns the smallest source-unit timestamp whose downcast is greater than or equal to +/// `target_value`. +/// +/// DataFusion timestamp downcasts truncate toward zero. For non-positive buckets that means the +/// bucket starts before `target_value * ratio`, so `<= x` can be rewritten as `< lower_bound(x+1)` +/// without dropping rows near zero or across negative boundaries. +fn lower_bound_for_ge( + target_value: i64, + source_unit: ArrowTimeUnit, + target_unit: ArrowTimeUnit, +) -> Option { + let ratio = finer_to_coarser_ratio(source_unit, target_unit)?; + let base = target_value.checked_mul(ratio)?; + if target_value <= 0 { + base.checked_sub(ratio - 1) + } else { + Some(base) + } +} + +fn timestamp_scalar_value(value: &ScalarValue) -> Option { + match value { + ScalarValue::TimestampSecond(Some(value), _) + | ScalarValue::TimestampMillisecond(Some(value), _) + | ScalarValue::TimestampMicrosecond(Some(value), _) + | ScalarValue::TimestampNanosecond(Some(value), _) => Some(*value), + _ => None, + } +} + +fn timestamp_scalar(unit: ArrowTimeUnit, timezone: Option>, value: i64) -> ScalarValue { + match unit { + ArrowTimeUnit::Second => ScalarValue::TimestampSecond(Some(value), timezone), + ArrowTimeUnit::Millisecond => ScalarValue::TimestampMillisecond(Some(value), timezone), + ArrowTimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(Some(value), timezone), + ArrowTimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(Some(value), timezone), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit}; + use async_trait::async_trait; + use common_time::Timestamp; + use common_time::range::TimestampRange; + use common_time::timestamp::TimeUnit; + use datafusion::catalog::Session; + use datafusion::config::ConfigOptions; + use datafusion::datasource::{TableProvider, provider_as_source}; + use datafusion::physical_plan::ExecutionPlan; + use datafusion_common::arrow::datatypes::Field; + use datafusion_common::{DFSchema, ScalarValue, ToDFSchema}; + use datafusion_expr::expr::{Between, Like}; + use datafusion_expr::expr_fn::{cast, col, try_cast}; + use datafusion_expr::{ + Expr, LogicalPlan, LogicalPlanBuilder, TableProviderFilterPushDown, TableScan, TableSource, + TableType, lit, + }; + use datafusion_optimizer::analyzer::AnalyzerRule; + use datafusion_optimizer::optimizer::{Optimizer, OptimizerContext}; + use datafusion_optimizer::push_down_filter::PushDownFilter; + use table::predicate::build_time_range_predicate; + + use super::{ + ConstNormalizationRule, PatternMatchKind, lower_bound_for_ge, try_cast_literal_to_type, + }; + + #[test] + fn test_normalize_direct_integer_cast_comparison() { + assert_filter_plan( + vec![Field::new("v", DataType::Int32, false)], + cast(col("v"), DataType::Int64).gt_eq(lit(42_i64)), + "Filter: t.v >= Int32(42)\n TableScan: t", + ); + } + + #[test] + fn test_normalize_non_column_operand() { + assert_filter_plan( + vec![Field::new("v", DataType::Int32, false)], + cast(col("v") + lit(1_i32), DataType::Int64).gt_eq(lit(42_i64)), + "Filter: t.v + Int32(1) >= Int32(42)\n TableScan: t", + ); + } + + #[test] + fn test_normalize_swapped_binary_comparison() { + assert_filter_plan( + vec![Field::new("v", DataType::Int16, false)], + lit(42_i64).lt_eq(cast(col("v"), DataType::Int64)), + "Filter: t.v >= Int16(42)\n TableScan: t", + ); + } + + #[test] + fn test_normalize_try_cast_target() { + assert_filter_plan( + vec![Field::new("v", DataType::Int16, false)], + try_cast(col("v"), DataType::Int64).gt_eq(lit(42_i64)), + "Filter: t.v >= Int16(42)\n TableScan: t", + ); + } + + #[test] + fn test_normalize_casted_constants() { + let fields = vec![Field::new("v", DataType::Int16, false)]; + let cases = [ + ( + col("v").gt_eq(cast(lit(42_i8), DataType::Int64)), + "Filter: t.v >= Int16(42)\n TableScan: t", + ), + ( + col("v").in_list( + vec![ + cast(lit(1_i8), DataType::Int64), + try_cast(lit(2_i8), DataType::Int64), + ], + false, + ), + "Filter: t.v IN ([Int16(1), Int16(2)])\n TableScan: t", + ), + ]; + + for (predicate, expected) in cases { + assert_filter_plan(fields.clone(), predicate, expected); + } + } + + #[test] + fn test_normalize_plain_integer_literals() { + let fields = vec![Field::new("v", DataType::Int16, false)]; + let cases = [ + ( + col("v").gt_eq(lit(42_i64)), + "Filter: t.v >= Int16(42)\n TableScan: t", + ), + ( + col("v").in_list(vec![lit(1_i64), lit(2_i64)], false), + "Filter: t.v IN ([Int16(1), Int16(2)])\n TableScan: t", + ), + ( + col("v").between(lit(3_i64), lit(5_i64)), + "Filter: t.v BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + ), + ]; + + for (predicate, expected) in cases { + assert_filter_plan(fields.clone(), predicate, expected); + } + } + + #[test] + fn test_normalize_unsigned_to_signed_literals() { + let cases = [ + ( + vec![Field::new("v", DataType::UInt8, false)], + cast(col("v"), DataType::Int16).lt_eq(lit(255_i16)), + "Filter: t.v <= UInt8(255)\n TableScan: t", + ), + ( + vec![Field::new("v", DataType::UInt16, false)], + cast(col("v"), DataType::Int32).gt_eq(lit(42_i32)), + "Filter: t.v >= UInt16(42)\n TableScan: t", + ), + ( + vec![Field::new("v", DataType::UInt32, false)], + cast(col("v"), DataType::Int64).between(lit(3_i64), lit(5_i64)), + "Filter: t.v BETWEEN UInt32(3) AND UInt32(5)\n TableScan: t", + ), + ]; + + for (fields, predicate, expected) in cases { + assert_filter_plan(fields, predicate, expected); + } + } + + #[test] + fn test_normalize_in_list_and_between() { + let fields = vec![Field::new("v", DataType::Int16, false)]; + let cases = [ + ( + cast(col("v"), DataType::Int64).in_list(vec![lit(1_i64), lit(2_i64)], false), + "Filter: t.v IN ([Int16(1), Int16(2)])\n TableScan: t", + ), + ( + cast(col("v"), DataType::Int64).between(lit(3_i64), lit(5_i64)), + "Filter: t.v BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + ), + ]; + + for (predicate, expected) in cases { + assert_filter_plan(fields.clone(), predicate, expected); + } + } + + #[test] + fn test_keep_non_lossless_literal_unchanged() { + assert_filter_plan( + vec![Field::new("v", DataType::Int16, false)], + col("v").gt_eq(lit(100_000_i64)), + "Filter: t.v >= Int64(100000)\n TableScan: t", + ); + } + + #[test] + fn test_normalize_scan_filters() { + let scan = build_scan_plan(test_schema(vec![Field::new("v", DataType::Int16, false)])); + let LogicalPlan::TableScan(scan) = scan else { + panic!("expected table scan"); + }; + let plan = LogicalPlan::TableScan(TableScan { + filters: vec![cast(col("v"), DataType::Int64).gt_eq(lit(42_i64))], + ..scan + }); + + let analyzed = analyze_plan(plan); + + assert_eq!( + vec![col("v").gt_eq(lit(42_i16))], + extract_scan_filters(&analyzed) + ); + } + + #[test] + fn test_normalize_negated_between() { + assert_filter_plan( + vec![Field::new("v", DataType::Int16, false)], + Expr::Between(Between { + expr: Box::new(cast(col("v"), DataType::Int64)), + negated: true, + low: Box::new(lit(3_i64)), + high: Box::new(lit(5_i64)), + }), + "Filter: t.v NOT BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + ); + } + + #[test] + fn test_normalize_like_literal() { + assert_pattern_match_plan( + PatternMatchKind::Like, + ScalarValue::LargeUtf8(Some("api%".to_string())), + "Filter: t.s LIKE Utf8(\"api%\")\n TableScan: t", + ); + } + + #[test] + fn test_normalize_similar_to_literal() { + assert_pattern_match_plan( + PatternMatchKind::SimilarTo, + ScalarValue::LargeUtf8(Some("api.*".to_string())), + "Filter: t.s SIMILAR TO Utf8(\"api.*\")\n TableScan: t", + ); + } + + #[test] + fn test_normalize_direct_timestamp_filter() { + assert_timestamp_pushdown( + vec![ + Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + ), + Field::new("tag", DataType::Utf8, true), + ], + ts_cast_to_ms() + .gt_eq(ts_ms_literal(-299_999)) + .and(ts_cast_to_ms().lt_eq(ts_ms_literal(10_000))) + .and(col("tag").eq(lit("api"))), + "Filter: t.ts >= TimestampNanosecond(-299999999999, None) AND t.ts < TimestampNanosecond(10001000000, None) AND t.tag = Utf8(\"api\")\n TableScan: t", + "TableScan: t, full_filters=[t.ts >= TimestampNanosecond(-299999999999, None), t.ts < TimestampNanosecond(10001000000, None), t.tag = Utf8(\"api\")]", + TimestampRange::new_inclusive( + Some(Timestamp::new_nanosecond(-299_999_999_999)), + Some(Timestamp::new_nanosecond(10_000_999_999)), + ), + ); + } + + #[test] + fn test_normalize_timestamp_between_filter() { + assert_timestamp_pushdown( + vec![Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )], + ts_cast_to_ms().between(ts_ms_literal(-299_999), ts_ms_literal(10_000)), + "Filter: t.ts >= TimestampNanosecond(-299999999999, None) AND t.ts < TimestampNanosecond(10001000000, None)\n TableScan: t", + "TableScan: t, full_filters=[t.ts >= TimestampNanosecond(-299999999999, None), t.ts < TimestampNanosecond(10001000000, None)]", + TimestampRange::new_inclusive( + Some(Timestamp::new_nanosecond(-299_999_999_999)), + Some(Timestamp::new_nanosecond(10_000_999_999)), + ), + ); + } + + #[test] + fn test_normalize_strict_timestamp_filter() { + assert_timestamp_pushdown( + vec![Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )], + ts_cast_to_ms() + .gt(ts_ms_literal(10_000)) + .and(ts_cast_to_ms().lt(ts_ms_literal(20_000))), + "Filter: t.ts >= TimestampNanosecond(10001000000, None) AND t.ts < TimestampNanosecond(20000000000, None)\n TableScan: t", + "TableScan: t, full_filters=[t.ts >= TimestampNanosecond(10001000000, None), t.ts < TimestampNanosecond(20000000000, None)]", + TimestampRange::new_inclusive( + Some(Timestamp::new_nanosecond(10_001_000_000)), + Some(Timestamp::new_nanosecond(19_999_999_999)), + ), + ); + } + + #[test] + fn test_normalize_zero_boundary_timestamp_filter() { + let fields = vec![Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )]; + + assert_timestamp_pushdown( + fields.clone(), + ts_cast_to_ms().gt_eq(ts_ms_literal(0)), + "Filter: t.ts >= TimestampNanosecond(-999999, None)\n TableScan: t", + "TableScan: t, full_filters=[t.ts >= TimestampNanosecond(-999999, None)]", + TimestampRange::from_start(Timestamp::new_nanosecond(-999_999)), + ); + + assert_timestamp_pushdown( + fields.clone(), + ts_cast_to_ms().lt(ts_ms_literal(0)), + "Filter: t.ts < TimestampNanosecond(-999999, None)\n TableScan: t", + "TableScan: t, full_filters=[t.ts < TimestampNanosecond(-999999, None)]", + TimestampRange::until_end(Timestamp::new_nanosecond(-999_999), false), + ); + + assert_timestamp_pushdown( + fields, + ts_cast_to_ms().between(ts_ms_literal(0), ts_ms_literal(0)), + "Filter: t.ts >= TimestampNanosecond(-999999, None) AND t.ts < TimestampNanosecond(1000000, None)\n TableScan: t", + "TableScan: t, full_filters=[t.ts >= TimestampNanosecond(-999999, None), t.ts < TimestampNanosecond(1000000, None)]", + TimestampRange::new_inclusive( + Some(Timestamp::new_nanosecond(-999_999)), + Some(Timestamp::new_nanosecond(999_999)), + ), + ); + } + + #[test] + fn test_timestamp_downcast_contract_matches_datafusion_casts() { + let cases = [ + (-1_000_001, -1), + (-1_000_000, -1), + (-999_999, 0), + (-1, 0), + (0, 0), + (999_999, 0), + (1_000_000, 1), + ]; + + for (source, expected) in cases { + let casted = try_cast_literal_to_type( + &ScalarValue::TimestampNanosecond(Some(source), None), + &DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + ) + .unwrap(); + assert_eq!( + ScalarValue::TimestampMillisecond(Some(expected), None), + casted + ); + } + + assert_eq!( + Some(-1_999_999), + lower_bound_for_ge(-1, ArrowTimeUnit::Nanosecond, ArrowTimeUnit::Millisecond) + ); + assert_eq!( + Some(-999_999), + lower_bound_for_ge(0, ArrowTimeUnit::Nanosecond, ArrowTimeUnit::Millisecond) + ); + assert_eq!( + Some(1_000_000), + lower_bound_for_ge(1, ArrowTimeUnit::Nanosecond, ArrowTimeUnit::Millisecond) + ); + } + + #[test] + fn test_normalize_plain_timestamp_literals() { + assert_timestamp_pushdown( + vec![Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )], + col("ts") + .gt_eq(ts_ms_literal(-299_999)) + .and(col("ts").lt_eq(ts_ms_literal(10_000))), + "Filter: t.ts >= TimestampNanosecond(-299999000000, None) AND t.ts <= TimestampNanosecond(10000000000, None)\n TableScan: t", + "TableScan: t, full_filters=[t.ts >= TimestampNanosecond(-299999000000, None), t.ts <= TimestampNanosecond(10000000000, None)]", + TimestampRange::new_inclusive( + Some(Timestamp::new_nanosecond(-299_999_000_000)), + Some(Timestamp::new_nanosecond(10_000_000_000)), + ), + ); + } + + #[test] + fn test_keep_timestamp_upcast_filter_unchanged() { + assert_filter_plan( + vec![Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + )], + cast( + col("ts"), + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + ) + .gt_eq(lit(ScalarValue::TimestampNanosecond(Some(1), None))), + "Filter: CAST(t.ts AS Timestamp(ns)) >= TimestampNanosecond(1, None)\n TableScan: t", + ); + } + + fn assert_pattern_match_plan(kind: PatternMatchKind, pattern: ScalarValue, expected: &str) { + let predicate = match kind { + PatternMatchKind::Like => Expr::Like(Like::new( + false, + Box::new(cast(col("s"), DataType::LargeUtf8)), + Box::new(lit(pattern)), + None, + false, + )), + PatternMatchKind::SimilarTo => Expr::SimilarTo(Like::new( + false, + Box::new(cast(col("s"), DataType::LargeUtf8)), + Box::new(lit(pattern)), + None, + false, + )), + }; + + assert_filter_plan( + vec![Field::new("s", DataType::Utf8, false)], + predicate, + expected, + ); + } + + fn assert_filter_plan(fields: Vec, predicate: Expr, expected: &str) { + assert_eq!(expected, analyze_filter(fields, predicate).to_string()); + } + + fn assert_timestamp_pushdown( + fields: Vec, + predicate: Expr, + expected_analyzed: &str, + expected_pushed: &str, + expected_range: TimestampRange, + ) { + let analyzed = analyze_filter(fields, predicate); + assert_eq!(expected_analyzed, analyzed.to_string()); + + let pushed = push_down_filters(analyzed); + assert_eq!(expected_pushed, pushed.to_string()); + + let range = + build_time_range_predicate("ts", TimeUnit::Nanosecond, &extract_scan_filters(&pushed)); + assert_eq!(expected_range, range); + } + + fn analyze_filter(fields: Vec, predicate: Expr) -> LogicalPlan { + analyze_plan(build_filter_plan(test_schema(fields), predicate)) + } + + fn analyze_plan(plan: LogicalPlan) -> LogicalPlan { + ConstNormalizationRule + .analyze(plan, &ConfigOptions::default()) + .unwrap() + } + + fn build_filter_plan(schema: Arc, predicate: Expr) -> LogicalPlan { + LogicalPlanBuilder::scan("t", test_source(schema), None) + .unwrap() + .filter(predicate) + .unwrap() + .build() + .unwrap() + } + + fn build_scan_plan(schema: Arc) -> LogicalPlan { + LogicalPlanBuilder::scan("t", test_source(schema), None) + .unwrap() + .build() + .unwrap() + } + + fn push_down_filters(plan: LogicalPlan) -> LogicalPlan { + Optimizer::with_rules(vec![Arc::new(PushDownFilter::new())]) + .optimize(plan, &OptimizerContext::new(), |_, _| {}) + .unwrap() + } + + fn ts_cast_to_ms() -> Expr { + cast( + col("ts"), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + ) + } + + fn ts_ms_literal(value: i64) -> Expr { + lit(ScalarValue::TimestampMillisecond(Some(value), None)) + } + + fn extract_scan_filters(plan: &LogicalPlan) -> Vec { + match plan { + LogicalPlan::TableScan(scan) => scan.filters.clone(), + _ => plan + .inputs() + .into_iter() + .flat_map(extract_scan_filters) + .collect(), + } + } + + fn test_schema(fields: Vec) -> Arc { + arrow_schema::Schema::new(fields).to_dfschema_ref().unwrap() + } + + fn test_source(schema: Arc) -> Arc { + let table = ExactPushdownProvider { + schema: Arc::new(schema.as_ref().as_arrow().clone()), + }; + provider_as_source(Arc::new(table)) + } + + #[derive(Debug)] + struct ExactPushdownProvider { + schema: arrow_schema::SchemaRef, + } + + #[async_trait] + impl TableProvider for ExactPushdownProvider { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn schema(&self) -> arrow_schema::SchemaRef { + self.schema.clone() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> datafusion::error::Result> { + unreachable!("scan should not be called in const_normalization tests") + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> datafusion::error::Result> { + Ok(vec![TableProviderFilterPushDown::Exact; filters.len()]) + } + } +} diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index fb2c3ea423..45a5700781 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -59,6 +59,7 @@ use crate::dist_plan::{ }; use crate::metrics::{QUERY_MEMORY_POOL_REJECTED_TOTAL, QUERY_MEMORY_POOL_USAGE_BYTES}; use crate::optimizer::ExtensionAnalyzerRule; +use crate::optimizer::const_normalization::ConstNormalizationRule; use crate::optimizer::constant_term::MatchesConstantTermOptimizer; use crate::optimizer::count_nest_aggr::CountNestAggrRule; use crate::optimizer::count_wildcard::CountWildcardToTimeIndexRule; @@ -157,6 +158,7 @@ impl QueryEngineState { analyzer .rules .insert(0, Arc::new(CountWildcardToTimeIndexRule)); + analyzer.rules.push(Arc::new(ConstNormalizationRule)); // Add ApplyFunctionRewrites rule, // Note we cannot use `analyzer.add_function_rewrite` diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index d2b199eb08..65532d738b 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -74,6 +74,7 @@ TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE (elapsed_compute.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN VERBOSE (0, 10, '5s') test; +-+-+ @@ -90,6 +91,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | logical_plan after TranscribeAtatRule_| SAME TEXT AS ABOVE_| | logical_plan after resolve_grouping_function_| SAME TEXT AS ABOVE_| | logical_plan after type_coercion_| SAME TEXT AS ABOVE_| +| logical_plan after ConstNormalizationRule_| SAME TEXT AS ABOVE_| | logical_plan after DistPlannerAnalyzer_| Projection: test.i, test.j, test.k_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| @@ -218,6 +220,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; -- SQLNESS REPLACE (elapsed_compute.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; +-+-+ @@ -235,6 +238,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; | logical_plan after TranscribeAtatRule_| SAME TEXT AS ABOVE_| | logical_plan after resolve_grouping_function_| SAME TEXT AS ABOVE_| | logical_plan after type_coercion_| SAME TEXT AS ABOVE_| +| logical_plan after ConstNormalizationRule_| SAME TEXT AS ABOVE_| | logical_plan after DistPlannerAnalyzer_| Projection: series, test.k, test.j_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Projection: test.i AS series, test.k, test.j_| @@ -360,6 +364,394 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; |_|_| +-+-+ +CREATE TABLE test_nano(i DOUBLE, j TIMESTAMP(9) TIME INDEX, k STRING PRIMARY KEY); + +Affected Rows: 0 + +INSERT INTO test_nano VALUES (1, 1000000, "a"), (1, 1000000, "b"), (2, 2000000, "a"); + +Affected Rows: 3 + +-- explain at 0s, 5s and 10s for a nanosecond time index. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +TQL EXPLAIN (0, 10, '5s') test_nano; + ++---------------+-----------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | +| | PromSeriesDivide: tags=["k"] | +| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST | +| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j | +| | Projection: test_nano.i, test_nano.j, test_nano.k | +| | Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None) | +| | Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | TableScan: test_nano | +| | ]] | +| physical_plan | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | +| | PromSeriesDivideExec: tags=["k"] | +| | SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true] | +| | RepartitionExec: REDACTED +| | ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j] | +| | FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3] | +| | ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k] | +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------+ + +-- explain verbose at 0s, 5s and 10s for a nanosecond time index. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; + ++-+-+ +| plan_type_| plan_| ++-+-+ +| initial_logical_plan_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Filter: test_nano.j >= TimestampMillisecond(-299999, None) AND test_nano.j <= TimestampMillisecond(10000, None)_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_TableScan: test_nano_| +| logical_plan after apply_function_rewrites_| SAME TEXT AS ABOVE_| +| logical_plan after count_wildcard_to_time_index_rule_| SAME TEXT AS ABOVE_| +| logical_plan after StringNormalizationRule_| SAME TEXT AS ABOVE_| +| logical_plan after TranscribeAtatRule_| SAME TEXT AS ABOVE_| +| logical_plan after resolve_grouping_function_| SAME TEXT AS ABOVE_| +| logical_plan after type_coercion_| SAME TEXT AS ABOVE_| +| logical_plan after ConstNormalizationRule_| SAME TEXT AS ABOVE_| +| logical_plan after DistPlannerAnalyzer_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Filter: test_nano.j >= TimestampMillisecond(-299999, None) AND test_nano.j <= TimestampMillisecond(10000, None)_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_| +| analyzed_logical_plan_| SAME TEXT AS ABOVE_| +| logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_predicate_subquery_| SAME TEXT AS ABOVE_| +| logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_lateral_join_| SAME TEXT AS ABOVE_| +| logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_cross_join_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_limit_| SAME TEXT AS ABOVE_| +| logical_plan after propagate_empty_relation_| SAME TEXT AS ABOVE_| +| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_filter_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_| +| logical_plan after eliminate_group_by_constant_| SAME TEXT AS ABOVE_| +| logical_plan after common_sub_expression_eliminate_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_1 >= TimestampMillisecond(-299999, None) AND __common_expr_1 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_1 >= TimestampMillisecond(-299999, None) AND __common_expr_1 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| +| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| +| logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_predicate_subquery_| SAME TEXT AS ABOVE_| +| logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_lateral_join_| SAME TEXT AS ABOVE_| +| logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_cross_join_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_limit_| SAME TEXT AS ABOVE_| +| logical_plan after propagate_empty_relation_| SAME TEXT AS ABOVE_| +| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_filter_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_| +| logical_plan after eliminate_group_by_constant_| SAME TEXT AS ABOVE_| +| logical_plan after common_sub_expression_eliminate_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_1, test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_2 >= TimestampMillisecond(-299999, None) AND __common_expr_2 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_2 >= TimestampMillisecond(-299999, None) AND __common_expr_2 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| +| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| +| logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_predicate_subquery_| SAME TEXT AS ABOVE_| +| logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_lateral_join_| SAME TEXT AS ABOVE_| +| logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_cross_join_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_limit_| SAME TEXT AS ABOVE_| +| logical_plan after propagate_empty_relation_| SAME TEXT AS ABOVE_| +| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_filter_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_| +| logical_plan after eliminate_group_by_constant_| SAME TEXT AS ABOVE_| +| logical_plan after common_sub_expression_eliminate_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_2, test_nano.i, test_nano.j, test_nano.k_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| +| logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| +| logical_plan_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| +|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| +|_|_Filter: __common_expr_3 >= TimestampMillisecond(-299999, None) AND __common_expr_3 <= TimestampMillisecond(10000, None)_| +|_|_Projection: CAST(test_nano.j AS Timestamp(ms)) AS __common_expr_3, test_nano.i, test_nano.j, test_nano.k_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: test_nano_| +|_| ]]_| +| initial_physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_| +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| initial_physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] | +|_|_MergeScanExec: REDACTED +|_|_| +| initial_physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k], schema=[i:Float64;N, j:Timestamp(ns), k:Utf8;N]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, schema=[__common_expr_3:Timestamp(ms), i:Float64;N, j:Timestamp(ns), k:Utf8;N]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], schema=[__common_expr_3:Timestamp(ms), i:Float64;N, j:Timestamp(ns), k:Utf8;N]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after OutputRequirements_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| +|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_| +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_| +| physical_plan after join_selection_| SAME TEXT AS ABOVE_| +| physical_plan after LimitedDistinctAggregation_| SAME TEXT AS ABOVE_| +| physical_plan after FilterPushdown_| SAME TEXT AS ABOVE_| +| physical_plan after parallelize_scan_| SAME TEXT AS ABOVE_| +| physical_plan after PassDistributionRule_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceSorting_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| +|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after EnforceDistribution_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| +|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_RepartitionExec: REDACTED +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceSorting_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| +|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_RepartitionExec: REDACTED +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_ProjectionExec: expr=[i@1 as i, j@2 as j, k@3 as k]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_| +| physical_plan after ProjectionPushdown_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| +|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_RepartitionExec: REDACTED +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_RepartitionExec: REDACTED +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| +| physical_plan after LimitPushPastWindows_| SAME TEXT AS ABOVE_| +| physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| +| physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| +| physical_plan after PushdownSort_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureCooperative_| SAME TEXT AS ABOVE_| +| physical_plan after FilterPushdown(Post)_| SAME TEXT AS ABOVE_| +| physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_| +| physical_plan after MatchesConstantTerm_| SAME TEXT AS ABOVE_| +| physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_| +| physical_plan after SanityCheckPlan_| SAME TEXT AS ABOVE_| +| physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| +|_|_RepartitionExec: REDACTED +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_RepartitionExec: REDACTED +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] | +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_RepartitionExec: REDACTED +|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +|_|_FilterExec: __common_expr_3@0 >= -299999 AND __common_expr_3@0 <= 10000, projection=[i@1, j@2, k@3], schema=[i:Float64;N, j:Timestamp(ns), k:Utf8;N]_| +|_|_ProjectionExec: expr=[CAST(j@1 AS Timestamp(ms)) as __common_expr_3, i@0 as i, j@1 as j, k@2 as k], schema=[__common_expr_3:Timestamp(ms), i:Float64;N, j:Timestamp(ns), k:Utf8;N]_| +|_|_MergeScanExec: REDACTED +|_|_| ++-+-+ + +DROP TABLE test_nano; + +Affected Rows: 0 + DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.sql b/tests/cases/standalone/common/tql-explain-analyze/explain.sql index 5e296d4b6b..62685b0952 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.sql @@ -25,6 +25,7 @@ TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE (elapsed_compute.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN VERBOSE (0, 10, '5s') test; -- explain verbose at 0s, 5s and 10s. No point at 0s. @@ -33,6 +34,28 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; -- SQLNESS REPLACE (elapsed_compute.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; +CREATE TABLE test_nano(i DOUBLE, j TIMESTAMP(9) TIME INDEX, k STRING PRIMARY KEY); + +INSERT INTO test_nano VALUES (1, 1000000, "a"), (1, 1000000, "b"), (2, 2000000, "a"); + +-- explain at 0s, 5s and 10s for a nanosecond time index. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +TQL EXPLAIN (0, 10, '5s') test_nano; + +-- explain verbose at 0s, 5s and 10s for a nanosecond time index. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; + +DROP TABLE test_nano; + DROP TABLE test; diff --git a/tests/cases/standalone/optimizer/time_index_filter_pushdown.result b/tests/cases/standalone/optimizer/time_index_filter_pushdown.result index ea10cfc416..1f281c684a 100644 --- a/tests/cases/standalone/optimizer/time_index_filter_pushdown.result +++ b/tests/cases/standalone/optimizer/time_index_filter_pushdown.result @@ -4,6 +4,7 @@ CREATE TABLE IF NOT EXISTS `cpu` ( `rack` STRING NULL, `os` STRING NULL, + `usage_small` SMALLINT NULL, `usage_user` BIGINT NULL, `greptime_timestamp` TIMESTAMP(9) NOT NULL, TIME INDEX (`greptime_timestamp`), @@ -25,11 +26,11 @@ Affected Rows: 0 INSERT INTO cpu VALUES - ("1", "linux", 10, "2023-06-12 01:04:49"), - ("1", "linux", 15, "2023-06-12 01:04:50"), - ("3", "windows", 25, "2023-06-12 01:05:00"), - ("5", "mac", 30, "2023-06-12 01:03:00"), - ("7", "linux", 45, "2023-06-12 02:00:00"); + ("1", "linux", 10, 10, "2023-06-12 01:04:49"), + ("1", "linux", 15, 15, "2023-06-12 01:04:50"), + ("3", "windows", 25, 25, "2023-06-12 01:05:00"), + ("5", "mac", 30, 30, "2023-06-12 01:03:00"), + ("7", "linux", 45, 45, "2023-06-12 02:00:00"); Affected Rows: 5 @@ -44,14 +45,108 @@ ADMIN FLUSH_TABLE ('cpu'); INSERT INTO cpu VALUES - ("2", "linux", 20, "2023-06-12 01:04:51"), - ("2", "windows", 22, "2023-06-12 01:06:00"), - ("4", "mac", 12, "2023-06-12 00:59:00"), - ("6", "linux", 35, "2023-06-12 01:04:55"), - ("8", "windows", 50, "2023-06-12 02:10:00"); + ("2", "linux", 20, 20, "2023-06-12 01:04:51"), + ("2", "windows", 22, 22, "2023-06-12 01:06:00"), + ("4", "mac", 12, 12, "2023-06-12 00:59:00"), + ("6", "linux", 35, 35, "2023-06-12 01:04:55"), + ("8", "windows", 50, 50, "2023-06-12 02:10:00"); Affected Rows: 5 +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT + rack +FROM + cpu +WHERE + usage_small IN (10, 20); + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: cpu.rack | +| | Filter: cpu.usage_small = Int16(10) OR cpu.usage_small = Int16(20) | +| | TableScan: cpu | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT + rack +FROM + cpu +WHERE + usage_small BETWEEN 10 AND 20; + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: cpu.rack | +| | Filter: cpu.usage_small >= Int16(10) AND cpu.usage_small <= Int16(20) | +| | TableScan: cpu | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + +CREATE TABLE + IF NOT EXISTS `cpu_single` ( + `rack` STRING NULL, + `usage_small` SMALLINT NULL, + `greptime_timestamp` TIMESTAMP(9) NOT NULL, + TIME INDEX (`greptime_timestamp`), + ) ENGINE = mito +WITH + (append_mode = 'true', sst_format = 'flat'); + +Affected Rows: 0 + +INSERT INTO + cpu_single +VALUES + ("1", 10, "2023-06-12 01:04:49"), + ("2", 20, "2023-06-12 01:04:50"), + ("3", 25, "2023-06-12 01:05:00"); + +Affected Rows: 3 + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +EXPLAIN ANALYZE VERBOSE SELECT + rack +FROM + cpu_single +WHERE + 10 <= usage_small; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_FilterExec: usage_small@1 >= 10, projection=[rack@0] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_UnorderedScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["rack", "usage_small"], "filters": ["usage_small >= Int16(10)"], "flat_format": REDACTED, "REDACTED +|_|_|_| +|_|_| Total rows: 3_| ++-+-+-+ + +drop table cpu_single; + +Affected Rows: 0 + -- SQLNESS SORT_RESULT 3 1 select count(*) diff --git a/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql b/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql index 28f4180a2c..c0636e26a4 100644 --- a/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql +++ b/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql @@ -4,6 +4,7 @@ CREATE TABLE IF NOT EXISTS `cpu` ( `rack` STRING NULL, `os` STRING NULL, + `usage_small` SMALLINT NULL, `usage_user` BIGINT NULL, `greptime_timestamp` TIMESTAMP(9) NOT NULL, TIME INDEX (`greptime_timestamp`), @@ -23,22 +24,71 @@ WITH INSERT INTO cpu VALUES - ("1", "linux", 10, "2023-06-12 01:04:49"), - ("1", "linux", 15, "2023-06-12 01:04:50"), - ("3", "windows", 25, "2023-06-12 01:05:00"), - ("5", "mac", 30, "2023-06-12 01:03:00"), - ("7", "linux", 45, "2023-06-12 02:00:00"); + ("1", "linux", 10, 10, "2023-06-12 01:04:49"), + ("1", "linux", 15, 15, "2023-06-12 01:04:50"), + ("3", "windows", 25, 25, "2023-06-12 01:05:00"), + ("5", "mac", 30, 30, "2023-06-12 01:03:00"), + ("7", "linux", 45, 45, "2023-06-12 02:00:00"); ADMIN FLUSH_TABLE ('cpu'); INSERT INTO cpu VALUES - ("2", "linux", 20, "2023-06-12 01:04:51"), - ("2", "windows", 22, "2023-06-12 01:06:00"), - ("4", "mac", 12, "2023-06-12 00:59:00"), - ("6", "linux", 35, "2023-06-12 01:04:55"), - ("8", "windows", 50, "2023-06-12 02:10:00"); + ("2", "linux", 20, 20, "2023-06-12 01:04:51"), + ("2", "windows", 22, 22, "2023-06-12 01:06:00"), + ("4", "mac", 12, 12, "2023-06-12 00:59:00"), + ("6", "linux", 35, 35, "2023-06-12 01:04:55"), + ("8", "windows", 50, 50, "2023-06-12 02:10:00"); + +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT + rack +FROM + cpu +WHERE + usage_small IN (10, 20); + +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT + rack +FROM + cpu +WHERE + usage_small BETWEEN 10 AND 20; + +CREATE TABLE + IF NOT EXISTS `cpu_single` ( + `rack` STRING NULL, + `usage_small` SMALLINT NULL, + `greptime_timestamp` TIMESTAMP(9) NOT NULL, + TIME INDEX (`greptime_timestamp`), + ) ENGINE = mito +WITH + (append_mode = 'true', sst_format = 'flat'); + +INSERT INTO + cpu_single +VALUES + ("1", 10, "2023-06-12 01:04:49"), + ("2", 20, "2023-06-12 01:04:50"), + ("3", 25, "2023-06-12 01:05:00"); + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +EXPLAIN ANALYZE VERBOSE SELECT + rack +FROM + cpu_single +WHERE + 10 <= usage_small; + +drop table cpu_single; -- SQLNESS SORT_RESULT 3 1 select @@ -70,4 +120,4 @@ where group by os; -drop table cpu; \ No newline at end of file +drop table cpu; From 7d330cc4e6dc147406a30405d233a2c17844c75a Mon Sep 17 00:00:00 2001 From: fys <40801205+fengys1996@users.noreply.github.com> Date: Tue, 12 May 2026 17:37:11 +0800 Subject: [PATCH 2/6] fix(mito2): schema-safe inverted index pruning (#8089) * fix(mito2): skip inverted index on per-SST type mismatch to avoid false negatives * restore INDEX_APPLY_MEMORY_USAGE * fix: cr * fix: cr --- .../src/sst/index/inverted_index/applier.rs | 254 +++++++++++++----- .../index/inverted_index/applier/builder.rs | 37 +-- .../src/sst/index/inverted_index/creator.rs | 4 +- src/mito2/src/sst/parquet.rs | 11 +- src/mito2/src/sst/parquet/reader.rs | 99 ++++++- .../change_col_type_inverted_index.result | 52 ++++ .../alter/change_col_type_inverted_index.sql | 29 ++ 7 files changed, 382 insertions(+), 104 deletions(-) create mode 100644 tests/cases/standalone/common/alter/change_col_type_inverted_index.result create mode 100644 tests/cases/standalone/common/alter/change_col_type_inverted_index.sql diff --git a/src/mito2/src/sst/index/inverted_index/applier.rs b/src/mito2/src/sst/index/inverted_index/applier.rs index f4c80051dd..a75962ec66 100644 --- a/src/mito2/src/sst/index/inverted_index/applier.rs +++ b/src/mito2/src/sst/index/inverted_index/applier.rs @@ -20,15 +20,18 @@ use std::time::Instant; use common_base::range_read::RangeReader; use common_telemetry::warn; +use datatypes::data_type::ConcreteDataType; use index::inverted_index::format::reader::{InvertedIndexBlobReader, InvertedIndexReadMetrics}; use index::inverted_index::search::index_apply::{ - ApplyOutput, IndexApplier, IndexNotFoundStrategy, SearchContext, + ApplyOutput, IndexApplier, IndexNotFoundStrategy, PredicatesIndexApplier, SearchContext, }; use index::inverted_index::search::predicate::Predicate; +use index::target::IndexTarget; use object_store::ObjectStore; use puffin::puffin_manager::cache::PuffinMetadataCacheRef; use puffin::puffin_manager::{PuffinManager, PuffinReader}; use snafu::ResultExt; +use store_api::metadata::RegionMetadataRef; use store_api::region_request::PathType; use store_api::storage::ColumnId; @@ -37,7 +40,8 @@ use crate::cache::file_cache::{FileCacheRef, FileType, IndexKey}; use crate::cache::index::inverted_index::{CachedInvertedIndexBlobReader, InvertedIndexCacheRef}; use crate::cache::index::result_cache::PredicateKey; use crate::error::{ - ApplyInvertedIndexSnafu, MetadataSnafu, PuffinBuildReaderSnafu, PuffinReadBlobSnafu, Result, + ApplyInvertedIndexSnafu, BuildIndexApplierSnafu, MetadataSnafu, PuffinBuildReaderSnafu, + PuffinReadBlobSnafu, Result, }; use crate::metrics::{INDEX_APPLY_ELAPSED, INDEX_APPLY_MEMORY_USAGE}; use crate::sst::file::RegionIndexId; @@ -121,10 +125,6 @@ pub(crate) struct InvertedIndexApplier { /// The cache of index files. file_cache: Option, - /// Predefined index applier used to apply predicates to index files - /// and return the relevant row group ids for further scan. - index_applier: Box, - /// The puffin manager factory. puffin_manager_factory: PuffinManagerFactory, @@ -134,35 +134,49 @@ pub(crate) struct InvertedIndexApplier { /// Puffin metadata cache. puffin_metadata_cache: Option, - /// Predicate key. Used to identify the predicate and fetch result from cache. - predicate_key: PredicateKey, + /// All collected predicates. + predicates: BTreeMap>, + + /// Default apply plan built from all collected predicates. + default_plan: SstApplyPlan, + + /// Expected predicate column types from the latest region metadata. + expected_predicate_col_types: BTreeMap, } pub(crate) type InvertedIndexApplierRef = Arc; +#[derive(Clone)] +pub(crate) struct SstApplyPlan { + pub predicate_key: PredicateKey, + pub index_applier: Arc, +} + impl InvertedIndexApplier { /// Creates a new `InvertedIndexApplier`. pub fn new( table_dir: String, path_type: PathType, store: ObjectStore, - index_applier: Box, puffin_manager_factory: PuffinManagerFactory, predicates: BTreeMap>, - ) -> Self { - INDEX_APPLY_MEMORY_USAGE.add(index_applier.memory_usage() as i64); + expected_predicate_col_types: BTreeMap, + ) -> Result { + let default_plan = Self::build_apply_plan(&predicates)?; + INDEX_APPLY_MEMORY_USAGE.add(default_plan.index_applier.memory_usage() as i64); - Self { + Ok(Self { table_dir, path_type, store, file_cache: None, - index_applier, puffin_manager_factory, inverted_index_cache: None, puffin_metadata_cache: None, - predicate_key: PredicateKey::new_inverted(Arc::new(predicates)), - } + predicates, + default_plan, + expected_predicate_col_types, + }) } /// Sets the file cache. @@ -186,11 +200,12 @@ impl InvertedIndexApplier { self } - /// Applies predicates to the provided SST file id and returns the relevant row group ids. + /// Applies predicates to one SST file with the provided index applier. /// /// # Arguments /// * `file_id` - The region file ID to apply predicates to /// * `file_size_hint` - Optional hint for file size to avoid extra metadata reads + /// * `index_applier` - Inverted index applier produced by `plan_for_sst`. /// * `metrics` - Optional mutable reference to collect metrics on demand #[tracing::instrument( skip_all, @@ -200,6 +215,7 @@ impl InvertedIndexApplier { &self, file_id: RegionIndexId, file_size_hint: Option, + index_applier: &PredicatesIndexApplier, mut metrics: Option<&mut InvertedIndexApplyMetrics>, ) -> Result { let start = Instant::now(); @@ -231,7 +247,7 @@ impl InvertedIndexApplier { InvertedIndexBlobReader::new(blob), index_cache.clone(), ); - self.index_applier + index_applier .apply( context, &mut index_reader, @@ -243,7 +259,7 @@ impl InvertedIndexApplier { .context(ApplyInvertedIndexSnafu) } else { let mut index_reader = InvertedIndexBlobReader::new(blob); - self.index_applier + index_applier .apply( context, &mut index_reader, @@ -344,82 +360,141 @@ impl InvertedIndexApplier { .context(PuffinBuildReaderSnafu) } - /// Returns the predicate key. - pub fn predicate_key(&self) -> &PredicateKey { - &self.predicate_key + /// Builds a per-SST apply plan. + /// + /// Returns `None` when no compatible predicate remains for this SST. + pub fn plan_for_sst(&self, sst_metadata: &RegionMetadataRef) -> Result> { + let mut compatible_predicates = BTreeMap::new(); + let mut has_type_mismatch = false; + + for (col_id, expected) in &self.expected_predicate_col_types { + if let Some(sst_col) = sst_metadata.column_by_id(*col_id) + && sst_col.column_schema.data_type != *expected + { + has_type_mismatch = true; + continue; + } + + if let Some(predicates) = self.predicates.get(col_id) { + compatible_predicates.insert(*col_id, predicates.clone()); + } + } + + if compatible_predicates.is_empty() { + return Ok(None); + } + + if !has_type_mismatch { + return Ok(Some(self.default_plan.clone())); + } + + let plan = Self::build_apply_plan(&compatible_predicates)?; + Ok(Some(plan)) + } + + fn build_apply_plan( + predicates_by_col: &BTreeMap>, + ) -> Result { + let predicates = predicates_by_col + .iter() + .map(|(col_id, preds)| (format!("{}", IndexTarget::ColumnId(*col_id)), preds.clone())) + .collect(); + + let index_applier = + PredicatesIndexApplier::try_from(predicates).context(BuildIndexApplierSnafu)?; + + let predicate_key = PredicateKey::new_inverted(Arc::new(predicates_by_col.clone())); + Ok(SstApplyPlan { + predicate_key, + index_applier: Arc::new(index_applier), + }) } } impl Drop for InvertedIndexApplier { fn drop(&mut self) { - INDEX_APPLY_MEMORY_USAGE.sub(self.index_applier.memory_usage() as i64); + INDEX_APPLY_MEMORY_USAGE.sub(self.default_plan.index_applier.memory_usage() as i64); } } #[cfg(test)] mod tests { + use api::v1::SemanticType; + use datatypes::data_type::ConcreteDataType; + use datatypes::schema::ColumnSchema; use futures::io::Cursor; - use index::bitmap::Bitmap; - use index::inverted_index::search::index_apply::MockIndexApplier; + use index::inverted_index::search::predicate::RegexMatchPredicate; use object_store::services::Memory; use puffin::puffin_manager::PuffinWriter; - use store_api::storage::FileId; + use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder}; + use store_api::storage::{FileId, RegionId}; use super::*; use crate::sst::index::RegionFileId; #[tokio::test] - async fn test_index_applier_apply_basic() { + async fn test_plan_for_sst() { let (_d, puffin_manager_factory) = - PuffinManagerFactory::new_for_test_async("test_index_applier_apply_basic_").await; + PuffinManagerFactory::new_for_test_async("test_plan_for_sst_basic_").await; let object_store = ObjectStore::new(Memory::default()).unwrap().finish(); - let file_id = RegionFileId::new(0.into(), FileId::random()); - let index_id = RegionIndexId::new(file_id, 0); let table_dir = "table_dir".to_string(); - let puffin_manager = puffin_manager_factory.build( - object_store.clone(), - RegionFilePathFactory::new(table_dir.clone(), PathType::Bare), + let mut predicates = BTreeMap::new(); + predicates.insert( + 1, + vec![Predicate::RegexMatch(RegexMatchPredicate { + pattern: "foo".to_string(), + })], ); - let mut writer = puffin_manager.writer(&index_id).await.unwrap(); - writer - .put_blob( - INDEX_BLOB_TYPE, - Cursor::new(vec![]), - Default::default(), - Default::default(), - ) - .await - .unwrap(); - writer.finish().await.unwrap(); - - let mut mock_index_applier = MockIndexApplier::new(); - mock_index_applier.expect_memory_usage().returning(|| 100); - mock_index_applier.expect_apply().returning(|_, _, _| { - Ok(ApplyOutput { - matched_segment_ids: Bitmap::new_bitvec(), - total_row_count: 100, - segment_row_count: 10, - }) - }); + let expected_predicate_col_types = + BTreeMap::from_iter([(1, ConcreteDataType::string_datatype())]); let sst_index_applier = InvertedIndexApplier::new( - table_dir.clone(), + table_dir, PathType::Bare, object_store, - Box::new(mock_index_applier), puffin_manager_factory, - Default::default(), - ); - let output = sst_index_applier.apply(index_id, None, None).await.unwrap(); - assert_eq!( - output, - ApplyOutput { - matched_segment_ids: Bitmap::new_bitvec(), - total_row_count: 100, - segment_row_count: 10, - } + predicates, + expected_predicate_col_types, + ) + .unwrap(); + let plan = sst_index_applier + .plan_for_sst(&mock_region_metadata()) + .unwrap(); + assert!(plan.is_some()); + } + + #[tokio::test] + async fn test_plan_for_sst_type_mismatch() { + let (_d, puffin_manager_factory) = + PuffinManagerFactory::new_for_test_async("test_plan_for_sst_type_mismatch_").await; + let object_store = ObjectStore::new(Memory::default()).unwrap().finish(); + let table_dir = "table_dir".to_string(); + + let mut predicates = BTreeMap::new(); + predicates.insert( + 1, + vec![Predicate::RegexMatch(RegexMatchPredicate { + pattern: "foo".to_string(), + })], ); + // Column id 1 is String in `mock_region_metadata`, set expected type to Int64. + let expected_predicate_col_types = + BTreeMap::from_iter([(1, ConcreteDataType::int64_datatype())]); + + let sst_index_applier = InvertedIndexApplier::new( + table_dir, + PathType::Bare, + object_store, + puffin_manager_factory, + predicates, + expected_predicate_col_types, + ) + .unwrap(); + let plan = sst_index_applier + .plan_for_sst(&mock_region_metadata()) + .unwrap(); + assert!(plan.is_none()); } #[tokio::test] @@ -448,19 +523,52 @@ mod tests { .unwrap(); writer.finish().await.unwrap(); - let mut mock_index_applier = MockIndexApplier::new(); - mock_index_applier.expect_memory_usage().returning(|| 100); - mock_index_applier.expect_apply().never(); - + let mut predicates = BTreeMap::new(); + predicates.insert( + 1, + vec![Predicate::RegexMatch(RegexMatchPredicate { + pattern: "foo".to_string(), + })], + ); + let expected_predicate_col_types = + BTreeMap::from_iter([(1, ConcreteDataType::string_datatype())]); let sst_index_applier = InvertedIndexApplier::new( table_dir.clone(), PathType::Bare, object_store, - Box::new(mock_index_applier), puffin_manager_factory, - Default::default(), - ); - let res = sst_index_applier.apply(index_id, None, None).await; + predicates, + expected_predicate_col_types, + ) + .unwrap(); + let plan = sst_index_applier + .plan_for_sst(&mock_region_metadata()) + .unwrap() + .unwrap(); + let res = sst_index_applier + .apply(index_id, None, &plan.index_applier, None) + .await; assert!(format!("{:?}", res.unwrap_err()).contains("Blob not found")); } + + fn mock_region_metadata() -> RegionMetadataRef { + let mut builder = RegionMetadataBuilder::new(RegionId::new(1024, 1)); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("tag", ConcreteDataType::string_datatype(), false), + semantic_type: SemanticType::Tag, + column_id: 1, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 2, + }) + .primary_key(vec![1]); + Arc::new(builder.build().unwrap()) + } } diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder.rs b/src/mito2/src/sst/index/inverted_index/applier/builder.rs index 078d1837fb..4938352792 100644 --- a/src/mito2/src/sst/index/inverted_index/applier/builder.rs +++ b/src/mito2/src/sst/index/inverted_index/applier/builder.rs @@ -25,9 +25,7 @@ use datafusion_common::ScalarValue; use datafusion_expr::{BinaryExpr, Expr, Operator}; use datatypes::data_type::ConcreteDataType; use datatypes::value::Value; -use index::inverted_index::search::index_apply::PredicatesIndexApplier; use index::inverted_index::search::predicate::Predicate; -use index::target::IndexTarget; use mito_codec::index::IndexValueCodec; use mito_codec::row_converter::SortField; use object_store::ObjectStore; @@ -39,9 +37,7 @@ use store_api::storage::ColumnId; use crate::cache::file_cache::FileCacheRef; use crate::cache::index::inverted_index::InvertedIndexCacheRef; -use crate::error::{ - BuildIndexApplierSnafu, ColumnNotFoundSnafu, ConvertValueSnafu, EncodeSnafu, Result, -}; +use crate::error::{ColumnNotFoundSnafu, ConvertValueSnafu, EncodeSnafu, Result}; use crate::sst::index::inverted_index::applier::InvertedIndexApplier; use crate::sst::index::puffin_manager::PuffinManagerFactory; @@ -137,33 +133,38 @@ impl<'a> InvertedIndexApplierBuilder<'a> { return Ok(None); } - let predicates = self - .output - .iter() - .map(|(column_id, predicates)| { - ( - format!("{}", IndexTarget::ColumnId(*column_id)), - predicates.clone(), - ) - }) - .collect::>(); - let applier = PredicatesIndexApplier::try_from(predicates); + let expected_predicate_column_types = self.expected_predicate_column_types(); Ok(Some( InvertedIndexApplier::new( self.table_dir, self.path_type, self.object_store, - Box::new(applier.context(BuildIndexApplierSnafu)?), self.puffin_manager_factory, self.output, - ) + expected_predicate_column_types, + )? .with_file_cache(self.file_cache) .with_puffin_metadata_cache(self.puffin_metadata_cache) .with_index_cache(self.inverted_index_cache), )) } + /// Returns `(column_id, data_type)` pairs for predicate columns + /// collected in `self.output`. + /// + /// The data types are resolved from the latest region manifest. Columns + /// that no longer exist in the latest metadata are skipped. + fn expected_predicate_column_types(&self) -> BTreeMap { + self.output + .keys() + .filter_map(|col_id| { + let col = self.metadata.column_by_id(*col_id)?; + Some((*col_id, col.column_schema.data_type.clone())) + }) + .collect() + } + /// Recursively traverses expressions to collect predicates. /// Results are stored in `self.output`. fn traverse_and_collect(&mut self, expr: &Expr) { diff --git a/src/mito2/src/sst/index/inverted_index/creator.rs b/src/mito2/src/sst/index/inverted_index/creator.rs index 386ee11b9b..952303f33f 100644 --- a/src/mito2/src/sst/index/inverted_index/creator.rs +++ b/src/mito2/src/sst/index/inverted_index/creator.rs @@ -614,9 +614,11 @@ mod tests { .build(&[expr]) .unwrap() .unwrap(); + let sst_metadata = Arc::new(region_metadata.clone()); + let plan = applier.plan_for_sst(&sst_metadata).unwrap().unwrap(); Box::pin(async move { applier - .apply(index_id, None, None) + .apply(index_id, None, &plan.index_applier, None) .await .unwrap() .matched_segment_ids diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs index d8d1f91e3d..cd22d2e551 100644 --- a/src/mito2/src/sst/parquet.rs +++ b/src/mito2/src/sst/parquet.rs @@ -929,11 +929,14 @@ mod tests { assert_eq!(metrics.filter_metrics.rg_minmax_filtered, 3); assert_eq!(metrics.filter_metrics.rg_inverted_filtered, 0); assert_eq!(metrics.filter_metrics.rows_inverted_filtered, 30); + let plan = inverted_index_applier + .as_ref() + .unwrap() + .plan_for_sst(&metadata) + .unwrap() + .unwrap(); let cached = index_result_cache - .get( - inverted_index_applier.unwrap().predicate_key(), - handle.file_id().file_id(), - ) + .get(&plan.predicate_key, handle.file_id().file_id()) .unwrap(); // inverted index will search all row groups assert!(cached.contains_row_group(0)); diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs index fef9a48571..f03856a521 100644 --- a/src/mito2/src/sst/parquet/reader.rs +++ b/src/mito2/src/sst/parquet/reader.rs @@ -681,6 +681,7 @@ impl ParquetReaderBuilder { } self.prune_row_groups_by_inverted_index( + read_format.metadata(), row_group_size, num_row_groups, &mut output, @@ -807,6 +808,7 @@ impl ParquetReaderBuilder { /// the correctness of the index. async fn prune_row_groups_by_inverted_index( &self, + sst_metadata: &RegionMetadataRef, row_group_size: usize, num_row_groups: usize, output: &mut RowGroupSelection, @@ -825,12 +827,19 @@ impl ParquetReaderBuilder { &self.inverted_index_appliers[..] }; for index_applier in appliers.iter().flatten() { - let predicate_key = index_applier.predicate_key(); + let Ok(Some(plan)) = index_applier + .plan_for_sst(sst_metadata) + .inspect_err(|e| warn!(e; "failed to build compatible plan for sst")) + else { + continue; + }; + // Fast path: return early if the result is in the cache. - let cached = self - .cache_strategy - .index_result_cache() - .and_then(|cache| cache.get(predicate_key, self.file_handle.file_id().file_id())); + let cached = self.cache_strategy.index_result_cache().and_then(|cache| { + let file_id = self.file_handle.file_id().file_id(); + cache.get(&plan.predicate_key, file_id) + }); + if let Some(result) = cached.as_ref() && all_required_row_groups_searched(output, result) { @@ -847,9 +856,11 @@ impl ParquetReaderBuilder { .apply( self.file_handle.index_id(), Some(file_size_hint), + &plan.index_applier, metrics.inverted_index_apply_metrics.as_mut(), ) .await; + let selection = match apply_res { Ok(apply_output) => RowGroupSelection::from_inverted_index_apply_output( row_group_size, @@ -863,7 +874,7 @@ impl ParquetReaderBuilder { }; self.apply_index_result_and_update_cache( - predicate_key, + &plan.predicate_key, self.file_handle.file_id().file_id(), selection, output, @@ -1832,8 +1843,21 @@ impl SimpleFilterContext { match sst_meta.column_by_id(column.column_id) { Some(sst_column) => { debug_assert_eq!(column.semantic_type, sst_column.semantic_type); - - (column, MaybeFilter::Filter(filter)) + // Schema evolution can make field columns with the same id have + // different concrete data types across SSTs. In that case, + // evaluating this simple filter against current SST column may + // raise an invalid cross-type comparison error (e.g. Float64 == Utf8). + let maybe_filter = if sst_column.column_schema.data_type + == column.column_schema.data_type + { + MaybeFilter::Filter(filter) + } else { + // Altering tag or timestamp column types is not allowed, + // so only field columns can reach this branch. + debug_assert_eq!(column.semantic_type, SemanticType::Field); + return None; + }; + (column, maybe_filter) } None => { // If the column is not present in the SST metadata, we evaluate the filter @@ -2162,6 +2186,7 @@ mod tests { use parquet::arrow::ArrowWriter; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; use store_api::region_request::PathType; + use store_api::storage::RegionId; use table::predicate::Predicate; use super::*; @@ -2326,6 +2351,64 @@ mod tests { )); } + #[test] + fn test_simple_filter_context_drops_mismatched_field_filter() { + let (sst_metadata, latest_metadata) = mock_metadata(); + let ctx = SimpleFilterContext::new_opt( + &sst_metadata, + Some(latest_metadata.as_ref()), + &col("field_0").eq(lit(1_i64)), + ); + + assert!(ctx.is_none()); + } + + fn mock_metadata() -> (RegionMetadataRef, RegionMetadataRef) { + let region_id = RegionId::new(1, 1); + let make_tag_0 = || ColumnMetadata { + column_schema: ColumnSchema::new( + "tag_0".to_string(), + ConcreteDataType::string_datatype(), + true, + ), + semantic_type: SemanticType::Tag, + column_id: 0, + }; + let make_ts = || ColumnMetadata { + column_schema: ColumnSchema::new( + "ts".to_string(), + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 2, + }; + let make_field_0 = |data_type| ColumnMetadata { + column_schema: ColumnSchema::new("field_0".to_string(), data_type, true), + semantic_type: SemanticType::Field, + column_id: 1, + }; + + let mut sst_builder = RegionMetadataBuilder::new(region_id); + sst_builder + .push_column_metadata(make_tag_0()) + .push_column_metadata(make_field_0(ConcreteDataType::uint64_datatype())) + .push_column_metadata(make_ts()) + .primary_key(vec![0]); + let sst_metadata = Arc::new(sst_builder.build().unwrap()); + + let mut expected_builder = RegionMetadataBuilder::new(region_id); + expected_builder + .push_column_metadata(make_tag_0()) + .push_column_metadata(make_field_0(ConcreteDataType::int64_datatype())) + .push_column_metadata(make_ts()) + .primary_key(vec![0]); + + let expected_metadata = Arc::new(expected_builder.build().unwrap()); + + (sst_metadata, expected_metadata) + } + #[test] fn test_physical_filter_context_skips_renamed_column() { let metadata: RegionMetadataRef = Arc::new(sst_region_metadata()); diff --git a/tests/cases/standalone/common/alter/change_col_type_inverted_index.result b/tests/cases/standalone/common/alter/change_col_type_inverted_index.result new file mode 100644 index 0000000000..14d3d754a0 --- /dev/null +++ b/tests/cases/standalone/common/alter/change_col_type_inverted_index.result @@ -0,0 +1,52 @@ +-- Regression test for issue #8074 +-- https://github.com/GreptimeTeam/greptimedb/issues/8074 +CREATE TABLE monitoring_data ( + host STRING INVERTED INDEX, + `region` STRING, + cpu_usage DOUBLE INVERTED INDEX, + `timestamp` TIMESTAMP TIME INDEX +) WITH ('append_mode'='true'); + +Affected Rows: 0 + +INSERT INTO monitoring_data (host, region, cpu_usage, `timestamp`) VALUES +('web-01', 'us-east', 12.5, '2026-05-06 10:00:00'), +('web-02', 'us-east', 18.3, '2026-05-06 10:00:00'), +('web-03', 'us-east', 91.2, '2026-05-06 10:00:00'), +('web-04', 'us-west', 73.8, '2026-05-06 10:00:00'); + +Affected Rows: 4 + +INSERT INTO monitoring_data (host, region, cpu_usage, `timestamp`) VALUES +('web-01', 'us-east', 15.2, '2026-05-06 10:01:00'), +('web-02', 'us-east', 23.7, '2026-05-06 10:01:00'), +('web-03', 'us-east', 94.5, '2026-05-06 10:01:00'), +('web-04', 'us-west', 78.1, '2026-05-06 10:01:00'); + +Affected Rows: 4 + +ADMIN FLUSH_TABLE('monitoring_data'); + ++--------------------------------------+ +| ADMIN FLUSH_TABLE('monitoring_data') | ++--------------------------------------+ +| 0 | ++--------------------------------------+ + +ALTER TABLE monitoring_data +MODIFY COLUMN cpu_usage STRING; + +Affected Rows: 0 + +SELECT host FROM monitoring_data WHERE cpu_usage = '23.7' ORDER BY host; + ++--------+ +| host | ++--------+ +| web-02 | ++--------+ + +DROP TABLE monitoring_data; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/alter/change_col_type_inverted_index.sql b/tests/cases/standalone/common/alter/change_col_type_inverted_index.sql new file mode 100644 index 0000000000..7020157758 --- /dev/null +++ b/tests/cases/standalone/common/alter/change_col_type_inverted_index.sql @@ -0,0 +1,29 @@ +-- Regression test for issue #8074 +-- https://github.com/GreptimeTeam/greptimedb/issues/8074 +CREATE TABLE monitoring_data ( + host STRING INVERTED INDEX, + `region` STRING, + cpu_usage DOUBLE INVERTED INDEX, + `timestamp` TIMESTAMP TIME INDEX +) WITH ('append_mode'='true'); + +INSERT INTO monitoring_data (host, region, cpu_usage, `timestamp`) VALUES +('web-01', 'us-east', 12.5, '2026-05-06 10:00:00'), +('web-02', 'us-east', 18.3, '2026-05-06 10:00:00'), +('web-03', 'us-east', 91.2, '2026-05-06 10:00:00'), +('web-04', 'us-west', 73.8, '2026-05-06 10:00:00'); + +INSERT INTO monitoring_data (host, region, cpu_usage, `timestamp`) VALUES +('web-01', 'us-east', 15.2, '2026-05-06 10:01:00'), +('web-02', 'us-east', 23.7, '2026-05-06 10:01:00'), +('web-03', 'us-east', 94.5, '2026-05-06 10:01:00'), +('web-04', 'us-west', 78.1, '2026-05-06 10:01:00'); + +ADMIN FLUSH_TABLE('monitoring_data'); + +ALTER TABLE monitoring_data +MODIFY COLUMN cpu_usage STRING; + +SELECT host FROM monitoring_data WHERE cpu_usage = '23.7' ORDER BY host; + +DROP TABLE monitoring_data; From dd420e33fe7018870b944493a9ec68c6c1bfb989 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Tue, 12 May 2026 18:43:57 +0800 Subject: [PATCH 3/6] fix: add standalone flag in standalone tests (#8100) * fix: test Signed-off-by: shuiyisong * fix: fmt and cargo.lock Signed-off-by: shuiyisong --------- Signed-off-by: shuiyisong --- Cargo.lock | 1 + tests-integration/Cargo.toml | 1 + tests-integration/tests/http.rs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index e4918c386c..acfa25df68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14098,6 +14098,7 @@ dependencies = [ "common-grpc", "common-memory-manager", "common-meta", + "common-options", "common-procedure", "common-procedure-test", "common-query", diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 050a2e5b1a..43850e4ed3 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -39,6 +39,7 @@ common-frontend.workspace = true common-grpc.workspace = true common-memory-manager.workspace = true common-meta = { workspace = true, features = ["testing"] } +common-options.workspace = true common-procedure.workspace = true common-query.workspace = true common-recordbatch.workspace = true diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index dcf943db51..7fe12cc8b9 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -37,6 +37,7 @@ use common_frontend::slow_query_event::{ SLOW_QUERY_TABLE_NAME, SLOW_QUERY_TABLE_QUERY_COLUMN_NAME, }; use common_memory_manager::OnExhaustedPolicy; +use common_options::plugin_options::StandaloneFlag; use flate2::Compression; use flate2::write::GzEncoder; use log_query::{Context, Limit, LogQuery, TimeFilter}; @@ -283,6 +284,7 @@ pub async fn test_http_auth_from_standalone_user_provider_config() { let fe_opts = options.component.frontend_options(); let mut plugins = Plugins::new(); + plugins.insert(StandaloneFlag); plugins::setup_frontend_plugins(&mut plugins, &[], &fe_opts) .await .unwrap(); From 73c267e64150c351ee5b4bb556e690337cc0db0e Mon Sep 17 00:00:00 2001 From: QuakeWang <45645138+QuakeWang@users.noreply.github.com> Date: Tue, 12 May 2026 22:20:05 +0800 Subject: [PATCH 4/6] fix(mito): ignore compaction override in enum option validation (#8094) * fix(mito): ignore compaction override in enum option validation Signed-off-by: QuakeWang * test: cover compaction override without compaction type Signed-off-by: QuakeWang * fix(mito): short-circuit enum option validation Signed-off-by: QuakeWang --------- Signed-off-by: QuakeWang --- src/mito2/src/region/options.rs | 69 ++++++++++++++++--- .../common/create/create_with_options.result | 16 +++++ .../common/create/create_with_options.sql | 12 ++++ 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs index 7827800295..f9241bd287 100644 --- a/src/mito2/src/region/options.rs +++ b/src/mito2/src/region/options.rs @@ -38,6 +38,8 @@ use crate::memtable::partition_tree::{DEFAULT_FREEZE_THRESHOLD, DEFAULT_MAX_KEYS use crate::sst::FormatType; const DEFAULT_INDEX_SEGMENT_ROW_COUNT: usize = 1024; +const COMPACTION_TWCS_PREFIX: &str = "compaction.twcs."; +const MEMTABLE_PARTITION_TREE_PREFIX: &str = "memtable.partition_tree."; pub(crate) fn parse_wal_options( options_map: &HashMap, @@ -138,7 +140,8 @@ impl TryFrom<&HashMap> for RegionOptions { // See https://github.com/serde-rs/serde/issues/1626 let options: RegionOptionsWithoutEnum = serde_json::from_str(&json).context(JsonOptionsSnafu)?; - let has_compaction_type = validate_enum_options(options_map, "compaction.type")?; + let has_compaction_type = + validate_enum_options(options_map, "compaction.type", &[COMPACTION_TWCS_PREFIX])?; let compaction = if has_compaction_type { serde_json::from_str(&json).context(JsonOptionsSnafu)? } else { @@ -148,7 +151,11 @@ impl TryFrom<&HashMap> for RegionOptions { let wal_options = parse_wal_options(options_map).context(JsonOptionsSnafu)?; let index_options: IndexOptions = serde_json::from_str(&json).context(JsonOptionsSnafu)?; - let memtable = if validate_enum_options(options_map, "memtable.type")? { + let memtable = if validate_enum_options( + options_map, + "memtable.type", + &[MEMTABLE_PARTITION_TREE_PREFIX], + )? { Some(serde_json::from_str(&json).context(JsonOptionsSnafu)?) } else { None @@ -437,25 +444,36 @@ fn options_map_to_value(options: &HashMap) -> Value { // `#[serde(default)]` doesn't support enum (https://github.com/serde-rs/serde/issues/1799) so we // check the type key first. /// Validates whether the `options_map` has valid options for specific `enum_tag_key` -/// and returns `true` if the map contains enum options. +/// and returns `true` if the map contains the enum tag. +/// +/// Variant options must start with one of `enum_option_prefixes`. If variant options +/// are provided, the tagged enum type key must also be provided. fn validate_enum_options( options_map: &HashMap, enum_tag_key: &str, + enum_option_prefixes: &[&str], ) -> Result { - let enum_type = enum_tag_key.split('.').next().unwrap(); - let mut has_other_options = false; + let mut has_enum_options = false; let mut has_tag = false; for key in options_map.keys() { if key == enum_tag_key { has_tag = true; - } else if key.starts_with(enum_type) { - has_other_options = true; + } else if !has_enum_options + && enum_option_prefixes + .iter() + .any(|prefix| key.starts_with(prefix)) + { + has_enum_options = true; + } + + if has_tag && has_enum_options { + break; } } // If tag is not provided, then other options for the enum should not exist. ensure!( - has_tag || !has_other_options, + has_tag || !has_enum_options, InvalidRegionOptionsSnafu { reason: format!("missing key {} in options", enum_tag_key), } @@ -538,6 +556,34 @@ mod tests { assert_eq!(expect, options); } + #[test] + fn test_with_compaction_override_true_without_compaction_type() { + let map = make_map(&[(COMPACTION_OVERRIDE, "true")]); + let options = RegionOptions::try_from(&map).unwrap(); + let expect = RegionOptions { + compaction_override: true, + ..Default::default() + }; + assert_eq!(expect, options); + } + + #[test] + fn test_with_compaction_override_false_without_compaction_type() { + let map = make_map(&[(COMPACTION_OVERRIDE, "false")]); + let options = RegionOptions::try_from(&map).unwrap(); + assert_eq!(RegionOptions::default(), options); + } + + #[test] + fn test_compaction_twcs_options_still_require_compaction_type_with_override() { + let map = make_map(&[ + (COMPACTION_OVERRIDE, "true"), + ("compaction.twcs.time_window", "2h"), + ]); + let err = RegionOptions::try_from(&map).unwrap_err(); + assert_eq!(StatusCode::InvalidArguments, err.status_code()); + } + fn test_with_wal_options(wal_options: &WalOptions) -> bool { let encoded_wal_options = serde_json::to_string(&wal_options).unwrap(); let map = make_map(&[(WAL_OPTIONS_KEY, &encoded_wal_options)]); @@ -608,6 +654,13 @@ mod tests { assert_eq!(StatusCode::InvalidArguments, err.status_code()); } + #[test] + fn test_without_memtable_type() { + let map = make_map(&[("memtable.partition_tree.index_max_keys_per_shard", "2048")]); + let err = RegionOptions::try_from(&map).unwrap_err(); + assert_eq!(StatusCode::InvalidArguments, err.status_code()); + } + #[test] fn test_with_merge_mode() { let map = make_map(&[("merge_mode", "last_row")]); diff --git a/tests/cases/standalone/common/create/create_with_options.result b/tests/cases/standalone/common/create/create_with_options.result index bb350a78aa..7c76ac6c63 100644 --- a/tests/cases/standalone/common/create/create_with_options.result +++ b/tests/cases/standalone/common/create/create_with_options.result @@ -81,6 +81,22 @@ drop table test_mito_options; Affected Rows: 0 +create table if not exists test_compaction_override_without_type( + host string, + ts timestamp, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with('compaction.override'='true'); + +Affected Rows: 0 + +drop table test_compaction_override_without_type; + +Affected Rows: 0 + create table if not exists invalid_compaction( host string, ts timestamp, diff --git a/tests/cases/standalone/common/create/create_with_options.sql b/tests/cases/standalone/common/create/create_with_options.sql index 13824f76aa..2ab8cee1b8 100644 --- a/tests/cases/standalone/common/create/create_with_options.sql +++ b/tests/cases/standalone/common/create/create_with_options.sql @@ -67,6 +67,18 @@ with( drop table test_mito_options; +create table if not exists test_compaction_override_without_type( + host string, + ts timestamp, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with('compaction.override'='true'); + +drop table test_compaction_override_without_type; + create table if not exists invalid_compaction( host string, ts timestamp, From 117a460d3430cf992a2a6fef5e7bdc44ee678d1d Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Wed, 13 May 2026 10:06:21 +0800 Subject: [PATCH 5/6] chore: update dashboard (#8098) Signed-off-by: WenyXu --- .../dashboards/metrics/cluster/dashboard.json | 9598 +++++++++-------- .../dashboards/metrics/cluster/dashboard.md | 84 + .../dashboards/metrics/cluster/dashboard.yaml | 62 + .../metrics/standalone/dashboard.json | 9598 +++++++++-------- .../metrics/standalone/dashboard.md | 84 + .../metrics/standalone/dashboard.yaml | 62 + 6 files changed, 10364 insertions(+), 9124 deletions(-) diff --git a/grafana/dashboards/metrics/cluster/dashboard.json b/grafana/dashboards/metrics/cluster/dashboard.json index 918eb3c4e8..d56df06a3b 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.json +++ b/grafana/dashboards/metrics/cluster/dashboard.json @@ -25,7 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 34, + "id": 33, "links": [], "panels": [ { @@ -927,7 +927,7 @@ "type": "stat" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -935,226 +935,226 @@ "y": 9 }, "id": 275, - "panels": [ + "panels": [], + "title": "Ingestion", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 193, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "ingestion", + "range": true, + "refId": "C" + } + ], + "title": "Total Ingestion Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 193, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "editorMode": "code", - "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "ingestion", - "range": true, - "refId": "C" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 277, + "options": { + "legend": { + "calcs": [ + "mean" ], - "title": "Total Ingestion Rate", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "http_logs" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 277, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "http-logs", - "range": true, - "refId": "http_logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "prometheus-remote-write", - "range": true, - "refId": "prometheus-remote-write" - } - ], - "title": "Ingestion Rate by Type", - "type": "timeseries" + "editorMode": "code", + "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "prometheus-remote-write", + "range": true, + "refId": "prometheus-remote-write" } ], - "title": "Ingestion", - "type": "row" + "title": "Ingestion Rate by Type", + "type": "timeseries" }, { "collapsed": true, @@ -1162,7 +1162,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 22 }, "id": 276, "panels": [ @@ -1231,7 +1231,7 @@ "h": 6, "w": 24, "x": 0, - "y": 19 + "y": 246 }, "id": 255, "options": { @@ -1303,7 +1303,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 23 }, "id": 274, "panels": [ @@ -1371,7 +1371,7 @@ "h": 10, "w": 12, "x": 0, - "y": 20 + "y": 247 }, "id": 256, "options": { @@ -1486,7 +1486,7 @@ "h": 10, "w": 12, "x": 12, - "y": 20 + "y": 247 }, "id": 262, "options": { @@ -1603,7 +1603,7 @@ "h": 10, "w": 12, "x": 0, - "y": 30 + "y": 257 }, "id": 266, "options": { @@ -1718,7 +1718,7 @@ "h": 10, "w": 12, "x": 12, - "y": 30 + "y": 257 }, "id": 268, "options": { @@ -1835,7 +1835,7 @@ "h": 10, "w": 12, "x": 0, - "y": 40 + "y": 267 }, "id": 269, "options": { @@ -1950,7 +1950,7 @@ "h": 10, "w": 12, "x": 12, - "y": 40 + "y": 267 }, "id": 271, "options": { @@ -2067,7 +2067,7 @@ "h": 10, "w": 12, "x": 0, - "y": 50 + "y": 277 }, "id": 272, "options": { @@ -2182,7 +2182,7 @@ "h": 10, "w": 12, "x": 12, - "y": 50 + "y": 277 }, "id": 273, "options": { @@ -2245,7 +2245,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 24 }, "id": 280, "panels": [ @@ -2314,7 +2314,7 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 288 }, "id": 281, "options": { @@ -2415,7 +2415,7 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 288 }, "id": 282, "options": { @@ -2516,7 +2516,7 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 296 }, "id": 283, "options": { @@ -2617,7 +2617,7 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 296 }, "id": 284, "options": { @@ -2716,7 +2716,7 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 304 }, "id": 285, "options": { @@ -2817,7 +2817,7 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 304 }, "id": 286, "options": { @@ -2919,7 +2919,7 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 312 }, "id": 287, "options": { @@ -3020,7 +3020,7 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 312 }, "id": 288, "options": { @@ -3064,7 +3064,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 25 }, "id": 289, "panels": [ @@ -3133,7 +3133,7 @@ "h": 6, "w": 24, "x": 0, - "y": 62 + "y": 289 }, "id": 292, "options": { @@ -3234,7 +3234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 295 }, "id": 290, "options": { @@ -3335,7 +3335,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 295 }, "id": 291, "options": { @@ -3464,7 +3464,7 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 303 }, "id": 336, "options": { @@ -3518,2519 +3518,2520 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 26 }, "id": 293, - "panels": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 193 + }, + "id": 294, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 193 + }, + "id": 295, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Buffer per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 225 + }, + "id": 296, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Buffer per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ingestion size by row counts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 225 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flush QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 233 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flush OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stall per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 233 + }, + "id": 299, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stall per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 241 + }, + "id": 300, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 241 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 249 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 249 + }, + "id": 303, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction latency by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 257 + }, + "id": 304, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Elapsed Time per Instance by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 257 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 265 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", + "range": true, + "refId": "C" + } + ], + "title": "WAL write size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Cached Bytes per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 265 + }, + "id": 307, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cached Bytes per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing compaction task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 273 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_compaction_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Compaction", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Raft engine (local disk) log store sync latency, p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 273 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "WAL sync duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead log operations latency at p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 281 + }, + "id": 311, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Log Store op duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing flush task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 281 + }, + "id": 312, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_flush_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Flush", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction oinput output bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 289 + }, + "id": 335, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-input", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-output", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Input/Output Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", + "[minipc-1:4000]-[]-[bulk_extend]-AVG", + "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", + "[minipc-1:4000]-[]-[process_bulk_req]-AVG", + "[minipc-1:4000]-[]-[write_bulk]-AVG" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 289 + }, + "id": 337, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", + "range": true, + "refId": "B" + } + ], + "title": "Region Worker Handle Bulk Insert Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction oinput output bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 297 + }, + "id": 348, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-series", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders", + "range": true, + "refId": "B" + } + ], + "title": "Active Series and Field Builders Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Per-stage elapsed time for region worker to decode requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 297 + }, + "id": 338, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", + "range": true, + "refId": "B" + } + ], + "title": "Region Worker Convert Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "The local cache miss of the datanode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 305 + }, + "id": 349, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~\"$datanode\"}[$__rate_interval]))", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss", + "type": "timeseries" + } + ], "title": "Mito Engine", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Request QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 294, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Request OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Request P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 295, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Buffer per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 296, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Buffer per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ingestion size by row counts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 297, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Rows per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Flush QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 298, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", - "range": true, - "refId": "A" - } - ], - "title": "Flush OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stall per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 299, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Stall per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read Stage OPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 300, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read Stage P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 301, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stage P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 302, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction OPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 303, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Compaction OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction latency by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 304, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", - "range": true, - "refId": "B" - } - ], - "title": "Compaction Elapsed Time per Instance by Stage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 305, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", - "range": true, - "refId": "A" - } - ], - "title": "Compaction P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 306, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", - "range": true, - "refId": "C" - } - ], - "title": "WAL write size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Cached Bytes per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 307, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Cached Bytes per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing compaction task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 308, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_compaction_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Compaction", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Raft engine (local disk) log store sync latency, p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 71 - }, - "id": 310, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "WAL sync duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead log operations latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 79 - }, - "id": 311, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Log Store op duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing flush task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 79 - }, - "id": 312, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_flush_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Flush", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction oinput output bytes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 87 - }, - "id": 335, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-input", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-output", - "range": true, - "refId": "B" - } - ], - "title": "Compaction Input/Output Bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[bulk_extend]-AVG", - "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[process_bulk_req]-AVG", - "[minipc-1:4000]-[]-[write_bulk]-AVG" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 87 - }, - "id": 337, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", - "range": true, - "refId": "B" - } - ], - "title": "Region Worker Handle Bulk Insert Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction oinput output bytes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 95 - }, - "id": 348, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-series", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders", - "range": true, - "refId": "B" - } - ], - "title": "Active Series and Field Builders Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Per-stage elapsed time for region worker to decode requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 95 - }, - "id": 338, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", - "range": true, - "refId": "B" - } - ], - "title": "Region Worker Convert Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "The local cache miss of the datanode.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 103 - }, - "id": 349, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~\"$datanode\"}[$__rate_interval]))", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Cache Miss", - "type": "timeseries" - }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 111 + "y": 27 }, "id": 313, "panels": [ @@ -6098,7 +6099,7 @@ "h": 10, "w": 24, "x": 0, - "y": 232 + "y": 459 }, "id": 314, "options": { @@ -6198,7 +6199,7 @@ "h": 7, "w": 12, "x": 0, - "y": 242 + "y": 469 }, "id": 315, "options": { @@ -6298,7 +6299,7 @@ "h": 7, "w": 12, "x": 12, - "y": 242 + "y": 469 }, "id": 316, "options": { @@ -6398,7 +6399,7 @@ "h": 7, "w": 12, "x": 0, - "y": 249 + "y": 476 }, "id": 317, "options": { @@ -6498,7 +6499,7 @@ "h": 7, "w": 12, "x": 12, - "y": 249 + "y": 476 }, "id": 318, "options": { @@ -6598,7 +6599,7 @@ "h": 7, "w": 12, "x": 0, - "y": 256 + "y": 483 }, "id": 319, "options": { @@ -6699,7 +6700,7 @@ "h": 7, "w": 12, "x": 12, - "y": 256 + "y": 483 }, "id": 320, "options": { @@ -6800,7 +6801,7 @@ "h": 7, "w": 12, "x": 0, - "y": 263 + "y": 490 }, "id": 321, "options": { @@ -6900,7 +6901,7 @@ "h": 7, "w": 12, "x": 12, - "y": 263 + "y": 490 }, "id": 322, "options": { @@ -7000,7 +7001,7 @@ "h": 7, "w": 12, "x": 0, - "y": 270 + "y": 497 }, "id": 323, "options": { @@ -7103,7 +7104,7 @@ "h": 7, "w": 12, "x": 12, - "y": 270 + "y": 497 }, "id": 334, "options": { @@ -7144,1827 +7145,1829 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 112 + "y": 28 }, "id": 351, - "panels": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region flush total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 195 + }, + "id": 350, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_flush_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region flush total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region checkpoint total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 195 + }, + "id": 352, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_checkpoint_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region checkpoint total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Topic estimated max replay size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 227 + }, + "id": 353, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_topic_estimated_replay_size", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Topic estimated replay size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Kafka logstore's bytes traffic", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 227 + }, + "id": 354, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", + "instant": false, + "legendFormat": "{{pod}}-{{logstore}}", + "range": true, + "refId": "A" + } + ], + "title": "Kafka logstore's bytes traffic", + "type": "timeseries" + } + ], "title": "Remote WAL", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Triggered region flush total", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 113 - }, - "id": 350, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "meta_triggered_region_flush_total", - "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Triggered region flush total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Triggered region checkpoint total", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 113 - }, - "id": 352, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "meta_triggered_region_checkpoint_total", - "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Triggered region checkpoint total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Topic estimated max replay size", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 121 - }, - "id": 353, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "meta_topic_estimated_replay_size", - "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Topic estimated replay size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Kafka logstore's bytes traffic", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 121 - }, - "id": 354, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", - "instant": false, - "legendFormat": "{{pod}}-{{logstore}}", - "range": true, - "refId": "A" - } - ], - "title": "Kafka logstore's bytes traffic", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 129 - }, - "id": 324, - "panels": [], - "title": "Metasrv", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Counter of region migration by source and destination", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisPlacement": "auto", - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 130 - }, - "id": 325, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", - "instant": false, - "legendFormat": "from-datanode-{{datanode_id}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", - "hide": false, - "instant": false, - "legendFormat": "to-datanode-{{datanode_id}}", - "range": true, - "refId": "B" - } - ], - "title": "Region migration datanode", - "type": "status-history" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Counter of region migration error", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 130 - }, - "id": 326, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_error", - "instant": false, - "legendFormat": "{{pod}}-{{state}}-{{error_type}}", - "range": true, - "refId": "A" - } - ], - "title": "Region migration error", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 138 - }, - "id": 327, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_datanode_load", - "instant": false, - "legendFormat": "Datanode-{{datanode_id}}-writeload", - "range": true, - "refId": "A" - } - ], - "title": "Datanode load", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 138 - }, - "id": 339, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])", - "instant": false, - "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ", - "range": true, - "refId": "A" - } - ], - "title": "Rate of SQL Executions (RDS)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Measures the response time of SQL executions via the RDS backend. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 146 - }, - "id": 340, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "SQL Execution Latency (RDS)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 146 - }, - "id": 341, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}} {{name}} p90", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Handler Execution Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 154 - }, - "id": 342, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Heartbeat Packet Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 154 - }, - "id": 345, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Meta Heartbeat Receive Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 162 - }, - "id": 343, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}-{{op}} p99", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Meta KV Ops Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 162 - }, - "id": 346, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}-{{op}} p99", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Rate of meta KV Ops", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 170 - }, - "id": 347, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateLogicalTables-{{step}} p90", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateTable-{{step}} p90", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateView-{{step}} p90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateFlow-{{step}} p90", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "DropTable-{{step}} p90", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "AlterTable-{{step}} p90", - "range": true, - "refId": "F", - "useBackend": false - } - ], - "title": "DDL Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Reconciliation stats", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 170 - }, - "id": 355, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_reconciliation_stats", - "hide": false, - "instant": false, - "legendFormat": "{{pod}}-{{table_type}}-{{type}}", - "range": true, - "refId": "A" - } - ], - "title": "Reconciliation stats", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed of Reconciliation steps", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 178 - }, - "id": 356, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", - "hide": false, - "instant": false, - "legendFormat": "{{procedure_name}}-{{step}}-P90", - "range": true, - "refId": "A" - } - ], - "title": "Reconciliation steps", - "type": "timeseries" - }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 186 + "y": 29 + }, + "id": 324, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration by source and destination", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 196 + }, + "id": 325, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", + "instant": false, + "legendFormat": "from-datanode-{{datanode_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", + "hide": false, + "instant": false, + "legendFormat": "to-datanode-{{datanode_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Region migration datanode", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration error", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 196 + }, + "id": 326, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_error", + "instant": false, + "legendFormat": "{{pod}}-{{state}}-{{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Region migration error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 228 + }, + "id": 327, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_datanode_load", + "instant": false, + "legendFormat": "Datanode-{{datanode_id}}-writeload", + "range": true, + "refId": "A" + } + ], + "title": "Datanode load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 228 + }, + "id": 339, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])", + "instant": false, + "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ", + "range": true, + "refId": "A" + } + ], + "title": "Rate of SQL Executions (RDS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Measures the response time of SQL executions via the RDS backend. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 236 + }, + "id": 340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "SQL Execution Latency (RDS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 236 + }, + "id": 341, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}} {{name}} p90", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Handler Execution Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 244 + }, + "id": 342, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Heartbeat Packet Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 244 + }, + "id": 345, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Meta Heartbeat Receive Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 252 + }, + "id": 343, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}-{{op}} p99", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Meta KV Ops Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 252 + }, + "id": 346, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}-{{op}} p99", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate of meta KV Ops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 347, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateLogicalTables-{{step}} p90", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateTable-{{step}} p90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateView-{{step}} p90", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateFlow-{{step}} p90", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "DropTable-{{step}} p90", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "AlterTable-{{step}} p90", + "range": true, + "refId": "F", + "useBackend": false + } + ], + "title": "DDL Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Reconciliation stats", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 355, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_reconciliation_stats", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}-{{table_type}}-{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconciliation stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed of Reconciliation steps", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 268 + }, + "id": 356, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", + "hide": false, + "instant": false, + "legendFormat": "{{procedure_name}}-{{step}}-P90", + "range": true, + "refId": "A" + } + ], + "title": "Reconciliation steps", + "type": "timeseries" + } + ], + "title": "Metasrv", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 }, "id": 328, "panels": [ @@ -9031,7 +9034,7 @@ "h": 8, "w": 12, "x": 0, - "y": 3055 + "y": 3282 }, "id": 329, "options": { @@ -9126,7 +9129,7 @@ "h": 8, "w": 12, "x": 12, - "y": 3055 + "y": 3282 }, "id": 330, "options": { @@ -9234,7 +9237,7 @@ "h": 8, "w": 9, "x": 0, - "y": 3063 + "y": 3290 }, "id": 331, "options": { @@ -9342,7 +9345,7 @@ "h": 8, "w": 9, "x": 9, - "y": 3063 + "y": 3290 }, "id": 332, "options": { @@ -9437,7 +9440,7 @@ "h": 8, "w": 6, "x": 18, - "y": 3063 + "y": 3290 }, "id": 333, "options": { @@ -9474,12 +9477,12 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 187 + "y": 31 }, "id": 357, "panels": [], @@ -9550,7 +9553,7 @@ "h": 8, "w": 24, "x": 0, - "y": 188 + "y": 32 }, "id": 358, "options": { @@ -9648,7 +9651,7 @@ "h": 8, "w": 12, "x": 0, - "y": 196 + "y": 40 }, "id": 359, "options": { @@ -9761,7 +9764,7 @@ "h": 8, "w": 12, "x": 12, - "y": 196 + "y": 40 }, "id": 360, "options": { @@ -9861,7 +9864,7 @@ "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 48 }, "id": 361, "options": { @@ -9974,7 +9977,7 @@ "h": 8, "w": 12, "x": 12, - "y": 204 + "y": 48 }, "id": 364, "options": { @@ -10074,7 +10077,7 @@ "h": 8, "w": 12, "x": 0, - "y": 212 + "y": 56 }, "id": 363, "options": { @@ -10187,7 +10190,7 @@ "h": 8, "w": 12, "x": 12, - "y": 212 + "y": 56 }, "id": 362, "options": { @@ -10221,6 +10224,477 @@ ], "title": "Save Alert Failure Rate", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 366, + "panels": [], + "title": "Hotspot", + "type": "row" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": false + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "disk_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 365, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [ + "region_rows", + "disk_size" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "table_schema" + } + ] + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Hotspot Regions", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write load of each datanode over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "total_region_rows", + "total_written_bytes_since_open", + "total_disk_size", + "total_sst_size", + "total_sst_num", + "total_memtable_size", + "total_manifest_size", + "total_index_size", + "cluster_disk_ratio_percent", + "cluster_sst_ratio_percent", + "cluster_rows_ratio_percent", + "max_region_disk_size", + "max_region_rows", + "leader_region_count", + "disk_ratio_percent", + "data_size" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 368, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "greptime_datanode_history_load", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Distribution of write load across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 369, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "greptime_datanode_history_load", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write) Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Distribution of leader regions and data size across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "total_region_rows", + "total_written_bytes_since_open", + "total_disk_size", + "total_sst_size", + "total_sst_num", + "total_memtable_size", + "total_manifest_size", + "total_index_size", + "cluster_disk_ratio_percent", + "cluster_sst_ratio_percent", + "cluster_rows_ratio_percent", + "max_region_disk_size", + "max_region_rows", + "leader_region_count", + "disk_ratio_percent", + "data_size" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 367, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^data_size$/", + "values": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Datanode Data Distribution", + "type": "piechart" } ], "preload": false, @@ -10354,12 +10828,12 @@ ] }, "time": { - "from": "now-1h", + "from": "now-2h", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "GreptimeDB", - "uid": "dejf3k5e7g2kgb", - "version": 15 -} + "title": "GreptimeDB Copy", + "uid": "dflfbxbwvvchsa", + "version": 37 +} \ No newline at end of file diff --git a/grafana/dashboards/metrics/cluster/dashboard.md b/grafana/dashboards/metrics/cluster/dashboard.md index c0bc49770b..c398ea7f90 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.md +++ b/grafana/dashboards/metrics/cluster/dashboard.md @@ -142,3 +142,87 @@ rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]) )` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` | | Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` | +# Hotspot +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Hotspot Regions | `WITH table_stats AS ( + SELECT + table_id, + COUNT(*) AS region_count, + SUM(disk_size) AS total_disk_size, + SUM(region_rows) as total_region_rows + FROM information_schema.region_statistics + WHERE region_role = 'Leader' + GROUP BY table_id + HAVING COUNT(*) > 1 +) + +SELECT + t.table_schema, + t.table_name, + + r.region_id, + t.table_id, + r.region_number, + + p.partition_description, + + + ROUND( + r.disk_size * 100.0 + / NULLIF(ts.total_disk_size, 0), + 2 + ) AS disk_size_share_percent, + + r.disk_size, + + ROUND( + r.region_rows * 100.0 + / NULLIF(ts.total_region_rows, 0), + 2 + ) AS region_rows_share_percent, + r.region_rows + +FROM information_schema.region_statistics r + +JOIN table_stats ts + ON r.table_id = ts.table_id + +JOIN information_schema.tables t + ON r.table_id = t.table_id + +LEFT JOIN information_schema.partitions p + ON p.table_schema = t.table_schema + AND p.table_name = t.table_name + AND p.greptime_partition_id = r.region_id + +WHERE r.region_role = 'Leader' + +ORDER BY region_rows_share_percent DESC +LIMIT 100;` | `table` | | `mysql` | -- | -- | +| Datanode Load(Write) | `greptime_datanode_history_load` | `timeseries` | Write load of each datanode over time. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` | +| Datanode Load(Write) Distribution | `greptime_datanode_history_load` | `piechart` | Distribution of write load across datanodes. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` | +| Datanode Data Distribution | `WITH leader_regions AS ( + SELECT + CONCAT( + 'datanode-', + p.peer_id, + ' (', + p.peer_addr, + ')' + ) AS datanode, + r.disk_size + FROM information_schema.region_statistics r + JOIN information_schema.region_peers p + ON r.region_id = p.region_id + WHERE r.region_role = 'Leader' + AND p.is_leader = 'Yes' +) + +SELECT + datanode, + COUNT(*) AS leader_region_count, + SUM(disk_size) AS data_size +FROM leader_regions +GROUP BY datanode +ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data size across datanodes. | `mysql` | `bytes` | -- | diff --git a/grafana/dashboards/metrics/cluster/dashboard.yaml b/grafana/dashboards/metrics/cluster/dashboard.yaml index 934cf58c0c..2617016083 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.yaml +++ b/grafana/dashboards/metrics/cluster/dashboard.yaml @@ -1153,3 +1153,65 @@ groups: type: prometheus uid: ${metrics} legendFormat: __auto + - title: Hotspot + panels: + - title: Hotspot Regions + type: table + queries: + - expr: "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;" + datasource: + type: mysql + uid: ${information_schema} + - title: Datanode Load(Write) + type: timeseries + description: Write load of each datanode over time. + unit: binBps + queries: + - expr: greptime_datanode_history_load + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-{{datanode_id}}({{instance}}) + - title: Datanode Load(Write) Distribution + type: piechart + description: Distribution of write load across datanodes. + unit: binBps + queries: + - expr: greptime_datanode_history_load + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-{{datanode_id}}({{instance}}) + - title: Datanode Data Distribution + type: piechart + description: Distribution of leader regions and data size across datanodes. + unit: bytes + queries: + - expr: |- + WITH leader_regions AS ( + SELECT + CONCAT( + 'datanode-', + p.peer_id, + ' (', + p.peer_addr, + ')' + ) AS datanode, + r.disk_size + FROM information_schema.region_statistics r + JOIN information_schema.region_peers p + ON r.region_id = p.region_id + WHERE r.region_role = 'Leader' + AND p.is_leader = 'Yes' + ) + + SELECT + datanode, + COUNT(*) AS leader_region_count, + SUM(disk_size) AS data_size + FROM leader_regions + GROUP BY datanode + ORDER BY data_size DESC; + datasource: + type: mysql + uid: ${information_schema} diff --git a/grafana/dashboards/metrics/standalone/dashboard.json b/grafana/dashboards/metrics/standalone/dashboard.json index 28f38f9dd7..229bdd30ea 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.json +++ b/grafana/dashboards/metrics/standalone/dashboard.json @@ -25,7 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 34, + "id": 33, "links": [], "panels": [ { @@ -927,7 +927,7 @@ "type": "stat" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -935,226 +935,226 @@ "y": 9 }, "id": 275, - "panels": [ + "panels": [], + "title": "Ingestion", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 193, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "ingestion", + "range": true, + "refId": "C" + } + ], + "title": "Total Ingestion Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 193, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "editorMode": "code", - "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "ingestion", - "range": true, - "refId": "C" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 277, + "options": { + "legend": { + "calcs": [ + "mean" ], - "title": "Total Ingestion Rate", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "http_logs" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 277, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "http-logs", - "range": true, - "refId": "http_logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "prometheus-remote-write", - "range": true, - "refId": "prometheus-remote-write" - } - ], - "title": "Ingestion Rate by Type", - "type": "timeseries" + "editorMode": "code", + "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "prometheus-remote-write", + "range": true, + "refId": "prometheus-remote-write" } ], - "title": "Ingestion", - "type": "row" + "title": "Ingestion Rate by Type", + "type": "timeseries" }, { "collapsed": true, @@ -1162,7 +1162,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 22 }, "id": 276, "panels": [ @@ -1231,7 +1231,7 @@ "h": 6, "w": 24, "x": 0, - "y": 19 + "y": 246 }, "id": 255, "options": { @@ -1303,7 +1303,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 23 }, "id": 274, "panels": [ @@ -1371,7 +1371,7 @@ "h": 10, "w": 12, "x": 0, - "y": 20 + "y": 247 }, "id": 256, "options": { @@ -1486,7 +1486,7 @@ "h": 10, "w": 12, "x": 12, - "y": 20 + "y": 247 }, "id": 262, "options": { @@ -1603,7 +1603,7 @@ "h": 10, "w": 12, "x": 0, - "y": 30 + "y": 257 }, "id": 266, "options": { @@ -1718,7 +1718,7 @@ "h": 10, "w": 12, "x": 12, - "y": 30 + "y": 257 }, "id": 268, "options": { @@ -1835,7 +1835,7 @@ "h": 10, "w": 12, "x": 0, - "y": 40 + "y": 267 }, "id": 269, "options": { @@ -1950,7 +1950,7 @@ "h": 10, "w": 12, "x": 12, - "y": 40 + "y": 267 }, "id": 271, "options": { @@ -2067,7 +2067,7 @@ "h": 10, "w": 12, "x": 0, - "y": 50 + "y": 277 }, "id": 272, "options": { @@ -2182,7 +2182,7 @@ "h": 10, "w": 12, "x": 12, - "y": 50 + "y": 277 }, "id": 273, "options": { @@ -2245,7 +2245,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 24 }, "id": 280, "panels": [ @@ -2314,7 +2314,7 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 288 }, "id": 281, "options": { @@ -2415,7 +2415,7 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 288 }, "id": 282, "options": { @@ -2516,7 +2516,7 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 296 }, "id": 283, "options": { @@ -2617,7 +2617,7 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 296 }, "id": 284, "options": { @@ -2716,7 +2716,7 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 304 }, "id": 285, "options": { @@ -2817,7 +2817,7 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 304 }, "id": 286, "options": { @@ -2919,7 +2919,7 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 312 }, "id": 287, "options": { @@ -3020,7 +3020,7 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 312 }, "id": 288, "options": { @@ -3064,7 +3064,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 25 }, "id": 289, "panels": [ @@ -3133,7 +3133,7 @@ "h": 6, "w": 24, "x": 0, - "y": 62 + "y": 289 }, "id": 292, "options": { @@ -3234,7 +3234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 295 }, "id": 290, "options": { @@ -3335,7 +3335,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 295 }, "id": 291, "options": { @@ -3464,7 +3464,7 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 303 }, "id": 336, "options": { @@ -3518,2519 +3518,2520 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 26 }, "id": 293, - "panels": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 193 + }, + "id": 294, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 193 + }, + "id": 295, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Buffer per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 225 + }, + "id": 296, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_write_buffer_bytes{}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Buffer per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ingestion size by row counts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 225 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flush QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 233 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flush OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stall per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 233 + }, + "id": 299, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stall per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 241 + }, + "id": 300, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 241 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 249 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 249 + }, + "id": 303, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction latency by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 257 + }, + "id": 304, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Elapsed Time per Instance by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 257 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 265 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", + "range": true, + "refId": "C" + } + ], + "title": "WAL write size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Cached Bytes per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 265 + }, + "id": 307, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_cache_bytes{}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cached Bytes per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing compaction task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 273 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_compaction_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Compaction", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Raft engine (local disk) log store sync latency, p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 273 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "WAL sync duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead log operations latency at p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 281 + }, + "id": 311, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Log Store op duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing flush task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 281 + }, + "id": 312, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_flush_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Flush", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction oinput output bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 289 + }, + "id": 335, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-input", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-output", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Input/Output Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", + "[minipc-1:4000]-[]-[bulk_extend]-AVG", + "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", + "[minipc-1:4000]-[]-[process_bulk_req]-AVG", + "[minipc-1:4000]-[]-[write_bulk]-AVG" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 289 + }, + "id": 337, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", + "range": true, + "refId": "B" + } + ], + "title": "Region Worker Handle Bulk Insert Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction oinput output bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 297 + }, + "id": 348, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-series", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders", + "range": true, + "refId": "B" + } + ], + "title": "Active Series and Field Builders Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Per-stage elapsed time for region worker to decode requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 297 + }, + "id": 338, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", + "range": true, + "refId": "B" + } + ], + "title": "Region Worker Convert Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "The local cache miss of the datanode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 305 + }, + "id": 349, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss", + "type": "timeseries" + } + ], "title": "Mito Engine", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Request QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 294, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Request OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Request P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 295, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Buffer per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 296, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_write_buffer_bytes{}", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Buffer per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ingestion size by row counts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 297, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Rows per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Flush QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 298, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", - "range": true, - "refId": "A" - } - ], - "title": "Flush OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stall per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 299, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Stall per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read Stage OPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 300, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read Stage P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 301, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stage P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 302, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction OPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 303, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Compaction OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction latency by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 304, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", - "range": true, - "refId": "B" - } - ], - "title": "Compaction Elapsed Time per Instance by Stage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 305, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", - "range": true, - "refId": "A" - } - ], - "title": "Compaction P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 306, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", - "range": true, - "refId": "C" - } - ], - "title": "WAL write size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Cached Bytes per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 307, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_cache_bytes{}", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Cached Bytes per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing compaction task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 308, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_compaction_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Compaction", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Raft engine (local disk) log store sync latency, p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 71 - }, - "id": 310, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "WAL sync duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead log operations latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 79 - }, - "id": 311, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Log Store op duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing flush task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 79 - }, - "id": 312, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_flush_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Flush", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction oinput output bytes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 87 - }, - "id": 335, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-input", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-output", - "range": true, - "refId": "B" - } - ], - "title": "Compaction Input/Output Bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[bulk_extend]-AVG", - "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[process_bulk_req]-AVG", - "[minipc-1:4000]-[]-[write_bulk]-AVG" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 87 - }, - "id": 337, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", - "range": true, - "refId": "B" - } - ], - "title": "Region Worker Handle Bulk Insert Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction oinput output bytes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 95 - }, - "id": 348, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-series", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders", - "range": true, - "refId": "B" - } - ], - "title": "Active Series and Field Builders Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Per-stage elapsed time for region worker to decode requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 95 - }, - "id": 338, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", - "range": true, - "refId": "B" - } - ], - "title": "Region Worker Convert Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "The local cache miss of the datanode.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 103 - }, - "id": 349, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", - "range": true, - "refId": "A" - } - ], - "title": "Cache Miss", - "type": "timeseries" - }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 111 + "y": 27 }, "id": 313, "panels": [ @@ -6098,7 +6099,7 @@ "h": 10, "w": 24, "x": 0, - "y": 232 + "y": 459 }, "id": 314, "options": { @@ -6198,7 +6199,7 @@ "h": 7, "w": 12, "x": 0, - "y": 242 + "y": 469 }, "id": 315, "options": { @@ -6298,7 +6299,7 @@ "h": 7, "w": 12, "x": 12, - "y": 242 + "y": 469 }, "id": 316, "options": { @@ -6398,7 +6399,7 @@ "h": 7, "w": 12, "x": 0, - "y": 249 + "y": 476 }, "id": 317, "options": { @@ -6498,7 +6499,7 @@ "h": 7, "w": 12, "x": 12, - "y": 249 + "y": 476 }, "id": 318, "options": { @@ -6598,7 +6599,7 @@ "h": 7, "w": 12, "x": 0, - "y": 256 + "y": 483 }, "id": 319, "options": { @@ -6699,7 +6700,7 @@ "h": 7, "w": 12, "x": 12, - "y": 256 + "y": 483 }, "id": 320, "options": { @@ -6800,7 +6801,7 @@ "h": 7, "w": 12, "x": 0, - "y": 263 + "y": 490 }, "id": 321, "options": { @@ -6900,7 +6901,7 @@ "h": 7, "w": 12, "x": 12, - "y": 263 + "y": 490 }, "id": 322, "options": { @@ -7000,7 +7001,7 @@ "h": 7, "w": 12, "x": 0, - "y": 270 + "y": 497 }, "id": 323, "options": { @@ -7103,7 +7104,7 @@ "h": 7, "w": 12, "x": 12, - "y": 270 + "y": 497 }, "id": 334, "options": { @@ -7144,1827 +7145,1829 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 112 + "y": 28 }, "id": 351, - "panels": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region flush total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 195 + }, + "id": 350, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_flush_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region flush total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region checkpoint total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 195 + }, + "id": 352, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_checkpoint_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region checkpoint total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Topic estimated max replay size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 227 + }, + "id": 353, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_topic_estimated_replay_size", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Topic estimated replay size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Kafka logstore's bytes traffic", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 227 + }, + "id": 354, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", + "instant": false, + "legendFormat": "{{pod}}-{{logstore}}", + "range": true, + "refId": "A" + } + ], + "title": "Kafka logstore's bytes traffic", + "type": "timeseries" + } + ], "title": "Remote WAL", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Triggered region flush total", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 113 - }, - "id": 350, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "meta_triggered_region_flush_total", - "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Triggered region flush total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Triggered region checkpoint total", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 113 - }, - "id": 352, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "meta_triggered_region_checkpoint_total", - "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Triggered region checkpoint total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Topic estimated max replay size", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 121 - }, - "id": 353, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "meta_topic_estimated_replay_size", - "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Topic estimated replay size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Kafka logstore's bytes traffic", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 121 - }, - "id": 354, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", - "instant": false, - "legendFormat": "{{pod}}-{{logstore}}", - "range": true, - "refId": "A" - } - ], - "title": "Kafka logstore's bytes traffic", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 129 - }, - "id": 324, - "panels": [], - "title": "Metasrv", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Counter of region migration by source and destination", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisPlacement": "auto", - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 130 - }, - "id": 325, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", - "instant": false, - "legendFormat": "from-datanode-{{datanode_id}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", - "hide": false, - "instant": false, - "legendFormat": "to-datanode-{{datanode_id}}", - "range": true, - "refId": "B" - } - ], - "title": "Region migration datanode", - "type": "status-history" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Counter of region migration error", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 130 - }, - "id": 326, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_error", - "instant": false, - "legendFormat": "{{pod}}-{{state}}-{{error_type}}", - "range": true, - "refId": "A" - } - ], - "title": "Region migration error", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 138 - }, - "id": 327, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_datanode_load", - "instant": false, - "legendFormat": "Datanode-{{datanode_id}}-writeload", - "range": true, - "refId": "A" - } - ], - "title": "Datanode load", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 138 - }, - "id": 339, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])", - "instant": false, - "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ", - "range": true, - "refId": "A" - } - ], - "title": "Rate of SQL Executions (RDS)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Measures the response time of SQL executions via the RDS backend. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 146 - }, - "id": 340, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "SQL Execution Latency (RDS)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 146 - }, - "id": 341, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}} {{name}} p90", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Handler Execution Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 154 - }, - "id": 342, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Heartbeat Packet Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 154 - }, - "id": 345, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Meta Heartbeat Receive Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 162 - }, - "id": 343, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}-{{op}} p99", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Meta KV Ops Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 162 - }, - "id": 346, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}}-{{op}} p99", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Rate of meta KV Ops", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 170 - }, - "id": 347, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateLogicalTables-{{step}} p90", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateTable-{{step}} p90", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateView-{{step}} p90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "CreateFlow-{{step}} p90", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "DropTable-{{step}} p90", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "AlterTable-{{step}} p90", - "range": true, - "refId": "F", - "useBackend": false - } - ], - "title": "DDL Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Reconciliation stats", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 170 - }, - "id": 355, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_meta_reconciliation_stats", - "hide": false, - "instant": false, - "legendFormat": "{{pod}}-{{table_type}}-{{type}}", - "range": true, - "refId": "A" - } - ], - "title": "Reconciliation stats", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed of Reconciliation steps", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 178 - }, - "id": 356, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", - "hide": false, - "instant": false, - "legendFormat": "{{procedure_name}}-{{step}}-P90", - "range": true, - "refId": "A" - } - ], - "title": "Reconciliation steps", - "type": "timeseries" - }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 186 + "y": 29 + }, + "id": 324, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration by source and destination", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 196 + }, + "id": 325, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", + "instant": false, + "legendFormat": "from-datanode-{{datanode_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", + "hide": false, + "instant": false, + "legendFormat": "to-datanode-{{datanode_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Region migration datanode", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration error", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 196 + }, + "id": 326, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_error", + "instant": false, + "legendFormat": "{{pod}}-{{state}}-{{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Region migration error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 228 + }, + "id": 327, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_datanode_load", + "instant": false, + "legendFormat": "Datanode-{{datanode_id}}-writeload", + "range": true, + "refId": "A" + } + ], + "title": "Datanode load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 228 + }, + "id": 339, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])", + "instant": false, + "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ", + "range": true, + "refId": "A" + } + ], + "title": "Rate of SQL Executions (RDS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Measures the response time of SQL executions via the RDS backend. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 236 + }, + "id": 340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "SQL Execution Latency (RDS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 236 + }, + "id": 341, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}} {{name}} p90", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Handler Execution Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 244 + }, + "id": 342, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Heartbeat Packet Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 244 + }, + "id": 345, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Meta Heartbeat Receive Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 252 + }, + "id": 343, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}-{{op}} p99", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Meta KV Ops Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 252 + }, + "id": 346, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}-{{op}} p99", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate of meta KV Ops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 347, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateLogicalTables-{{step}} p90", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateTable-{{step}} p90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateView-{{step}} p90", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "CreateFlow-{{step}} p90", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "DropTable-{{step}} p90", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "AlterTable-{{step}} p90", + "range": true, + "refId": "F", + "useBackend": false + } + ], + "title": "DDL Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Reconciliation stats", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 355, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_reconciliation_stats", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}-{{table_type}}-{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconciliation stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed of Reconciliation steps", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 268 + }, + "id": 356, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", + "hide": false, + "instant": false, + "legendFormat": "{{procedure_name}}-{{step}}-P90", + "range": true, + "refId": "A" + } + ], + "title": "Reconciliation steps", + "type": "timeseries" + } + ], + "title": "Metasrv", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 }, "id": 328, "panels": [ @@ -9031,7 +9034,7 @@ "h": 8, "w": 12, "x": 0, - "y": 3055 + "y": 3282 }, "id": 329, "options": { @@ -9126,7 +9129,7 @@ "h": 8, "w": 12, "x": 12, - "y": 3055 + "y": 3282 }, "id": 330, "options": { @@ -9234,7 +9237,7 @@ "h": 8, "w": 9, "x": 0, - "y": 3063 + "y": 3290 }, "id": 331, "options": { @@ -9342,7 +9345,7 @@ "h": 8, "w": 9, "x": 9, - "y": 3063 + "y": 3290 }, "id": 332, "options": { @@ -9437,7 +9440,7 @@ "h": 8, "w": 6, "x": 18, - "y": 3063 + "y": 3290 }, "id": 333, "options": { @@ -9474,12 +9477,12 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 187 + "y": 31 }, "id": 357, "panels": [], @@ -9550,7 +9553,7 @@ "h": 8, "w": 24, "x": 0, - "y": 188 + "y": 32 }, "id": 358, "options": { @@ -9648,7 +9651,7 @@ "h": 8, "w": 12, "x": 0, - "y": 196 + "y": 40 }, "id": 359, "options": { @@ -9761,7 +9764,7 @@ "h": 8, "w": 12, "x": 12, - "y": 196 + "y": 40 }, "id": 360, "options": { @@ -9861,7 +9864,7 @@ "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 48 }, "id": 361, "options": { @@ -9974,7 +9977,7 @@ "h": 8, "w": 12, "x": 12, - "y": 204 + "y": 48 }, "id": 364, "options": { @@ -10074,7 +10077,7 @@ "h": 8, "w": 12, "x": 0, - "y": 212 + "y": 56 }, "id": 363, "options": { @@ -10187,7 +10190,7 @@ "h": 8, "w": 12, "x": 12, - "y": 212 + "y": 56 }, "id": 362, "options": { @@ -10221,6 +10224,477 @@ ], "title": "Save Alert Failure Rate", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 366, + "panels": [], + "title": "Hotspot", + "type": "row" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": false + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "disk_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 365, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [ + "region_rows", + "disk_size" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "table_schema" + } + ] + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Hotspot Regions", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write load of each datanode over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "total_region_rows", + "total_written_bytes_since_open", + "total_disk_size", + "total_sst_size", + "total_sst_num", + "total_memtable_size", + "total_manifest_size", + "total_index_size", + "cluster_disk_ratio_percent", + "cluster_sst_ratio_percent", + "cluster_rows_ratio_percent", + "max_region_disk_size", + "max_region_rows", + "leader_region_count", + "disk_ratio_percent", + "data_size" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 368, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "greptime_datanode_history_load", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Distribution of write load across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 369, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "greptime_datanode_history_load", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write) Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Distribution of leader regions and data size across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "total_region_rows", + "total_written_bytes_since_open", + "total_disk_size", + "total_sst_size", + "total_sst_num", + "total_memtable_size", + "total_manifest_size", + "total_index_size", + "cluster_disk_ratio_percent", + "cluster_sst_ratio_percent", + "cluster_rows_ratio_percent", + "max_region_disk_size", + "max_region_rows", + "leader_region_count", + "disk_ratio_percent", + "data_size" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 367, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^data_size$/", + "values": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Datanode Data Distribution", + "type": "piechart" } ], "preload": false, @@ -10354,12 +10828,12 @@ ] }, "time": { - "from": "now-1h", + "from": "now-2h", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "GreptimeDB", - "uid": "dejf3k5e7g2kgb", - "version": 15 -} + "title": "GreptimeDB Copy", + "uid": "dflfbxbwvvchsa", + "version": 37 +} \ No newline at end of file diff --git a/grafana/dashboards/metrics/standalone/dashboard.md b/grafana/dashboards/metrics/standalone/dashboard.md index 63b45208c5..d2a4ddba0a 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.md +++ b/grafana/dashboards/metrics/standalone/dashboard.md @@ -142,3 +142,87 @@ rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]) )` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` | | Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` | +# Hotspot +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Hotspot Regions | `WITH table_stats AS ( + SELECT + table_id, + COUNT(*) AS region_count, + SUM(disk_size) AS total_disk_size, + SUM(region_rows) as total_region_rows + FROM information_schema.region_statistics + WHERE region_role = 'Leader' + GROUP BY table_id + HAVING COUNT(*) > 1 +) + +SELECT + t.table_schema, + t.table_name, + + r.region_id, + t.table_id, + r.region_number, + + p.partition_description, + + + ROUND( + r.disk_size * 100.0 + / NULLIF(ts.total_disk_size, 0), + 2 + ) AS disk_size_share_percent, + + r.disk_size, + + ROUND( + r.region_rows * 100.0 + / NULLIF(ts.total_region_rows, 0), + 2 + ) AS region_rows_share_percent, + r.region_rows + +FROM information_schema.region_statistics r + +JOIN table_stats ts + ON r.table_id = ts.table_id + +JOIN information_schema.tables t + ON r.table_id = t.table_id + +LEFT JOIN information_schema.partitions p + ON p.table_schema = t.table_schema + AND p.table_name = t.table_name + AND p.greptime_partition_id = r.region_id + +WHERE r.region_role = 'Leader' + +ORDER BY region_rows_share_percent DESC +LIMIT 100;` | `table` | | `mysql` | -- | -- | +| Datanode Load(Write) | `greptime_datanode_history_load` | `timeseries` | Write load of each datanode over time. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` | +| Datanode Load(Write) Distribution | `greptime_datanode_history_load` | `piechart` | Distribution of write load across datanodes. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` | +| Datanode Data Distribution | `WITH leader_regions AS ( + SELECT + CONCAT( + 'datanode-', + p.peer_id, + ' (', + p.peer_addr, + ')' + ) AS datanode, + r.disk_size + FROM information_schema.region_statistics r + JOIN information_schema.region_peers p + ON r.region_id = p.region_id + WHERE r.region_role = 'Leader' + AND p.is_leader = 'Yes' +) + +SELECT + datanode, + COUNT(*) AS leader_region_count, + SUM(disk_size) AS data_size +FROM leader_regions +GROUP BY datanode +ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data size across datanodes. | `mysql` | `bytes` | -- | diff --git a/grafana/dashboards/metrics/standalone/dashboard.yaml b/grafana/dashboards/metrics/standalone/dashboard.yaml index 32f771f66f..db1d2b6a7a 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.yaml +++ b/grafana/dashboards/metrics/standalone/dashboard.yaml @@ -1153,3 +1153,65 @@ groups: type: prometheus uid: ${metrics} legendFormat: __auto + - title: Hotspot + panels: + - title: Hotspot Regions + type: table + queries: + - expr: "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;" + datasource: + type: mysql + uid: ${information_schema} + - title: Datanode Load(Write) + type: timeseries + description: Write load of each datanode over time. + unit: binBps + queries: + - expr: greptime_datanode_history_load + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-{{datanode_id}}({{instance}}) + - title: Datanode Load(Write) Distribution + type: piechart + description: Distribution of write load across datanodes. + unit: binBps + queries: + - expr: greptime_datanode_history_load + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-{{datanode_id}}({{instance}}) + - title: Datanode Data Distribution + type: piechart + description: Distribution of leader regions and data size across datanodes. + unit: bytes + queries: + - expr: |- + WITH leader_regions AS ( + SELECT + CONCAT( + 'datanode-', + p.peer_id, + ' (', + p.peer_addr, + ')' + ) AS datanode, + r.disk_size + FROM information_schema.region_statistics r + JOIN information_schema.region_peers p + ON r.region_id = p.region_id + WHERE r.region_role = 'Leader' + AND p.is_leader = 'Yes' + ) + + SELECT + datanode, + COUNT(*) AS leader_region_count, + SUM(disk_size) AS data_size + FROM leader_regions + GROUP BY datanode + ORDER BY data_size DESC; + datasource: + type: mysql + uid: ${information_schema} From 0518567e5aeb415589ef36942bd85c37e666ec3c Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Tue, 12 May 2026 20:04:04 -0700 Subject: [PATCH 6/6] feat: add export-v2 snapshot listing (#8096) * feat: add export-v2 snapshot listing Signed-off-by: jeremyhi * fix: by AI comments Signed-off-by: jeremyhi * fix: allow export-v2 list at storage roots Signed-off-by: jeremyhi --------- Signed-off-by: jeremyhi --- src/cli/src/data/export_v2/command.rs | 307 +++++++++++++++++++++++++- src/cli/src/data/snapshot_storage.rs | 168 +++++++++++++- 2 files changed, 462 insertions(+), 13 deletions(-) diff --git a/src/cli/src/data/export_v2/command.rs b/src/cli/src/data/export_v2/command.rs index ddcb323fef..a7da6d3862 100644 --- a/src/cli/src/data/export_v2/command.rs +++ b/src/cli/src/data/export_v2/command.rs @@ -34,7 +34,7 @@ use crate::data::export_v2::error::{ }; use crate::data::export_v2::extractor::SchemaExtractor; use crate::data::export_v2::manifest::{ - ChunkMeta, DataFormat, MANIFEST_VERSION, Manifest, TimeRange, + ChunkMeta, DataFormat, MANIFEST_FILE, MANIFEST_VERSION, Manifest, TimeRange, }; use crate::data::path::ddl_path_for_schema; use crate::data::snapshot_storage::{OpenDalStorage, SnapshotStorage, validate_uri}; @@ -46,16 +46,73 @@ use crate::database::{DatabaseClient, parse_proxy_opts}; pub enum ExportV2Command { /// Create a new snapshot. Create(ExportCreateCommand), + /// List snapshots under a parent location. + List(ExportListCommand), } impl ExportV2Command { pub async fn build(&self) -> std::result::Result, BoxedError> { match self { ExportV2Command::Create(cmd) => cmd.build().await, + ExportV2Command::List(cmd) => cmd.build().await, } } } +/// List snapshots under a parent location. +#[derive(Debug, Parser)] +pub struct ExportListCommand { + /// Parent storage location whose direct subdirectories are snapshots. + #[clap(long)] + location: String, + + /// Object store configuration for remote storage backends. + #[clap(flatten)] + storage: ObjectStoreConfig, +} + +impl ExportListCommand { + pub async fn build(&self) -> std::result::Result, BoxedError> { + validate_uri(&self.location).map_err(BoxedError::new)?; + let storage = OpenDalStorage::from_parent_uri(&self.location, &self.storage) + .map_err(BoxedError::new)?; + + Ok(Box::new(ExportList { + location: self.location.clone(), + storage, + })) + } +} + +/// Export list tool implementation. +pub struct ExportList { + location: String, + storage: OpenDalStorage, +} + +#[async_trait] +impl Tool for ExportList { + async fn do_work(&self) -> std::result::Result<(), BoxedError> { + self.run().await.map_err(BoxedError::new) + } +} + +impl ExportList { + async fn run(&self) -> Result<()> { + let result = scan_snapshots(&self.storage).await?; + + println!("Scanning: {}", self.location); + if result.snapshots.is_empty() { + println!("No snapshots found."); + } else { + print_snapshot_list(&result.snapshots, result.unreadable.len()); + } + print_unreadable_warnings(&result.unreadable); + + Ok(()) + } +} + /// Create a new snapshot. #[derive(Debug, Parser)] pub struct ExportCreateCommand { @@ -628,10 +685,138 @@ fn format_chunk_plan(chunks: &[ChunkMeta]) -> String { format!("[{}]", items.join(", ")) } +#[derive(Debug)] +struct SnapshotListEntry { + path: String, + manifest: Manifest, +} + +#[derive(Debug, Default)] +struct SnapshotScanResult { + snapshots: Vec, + unreadable: Vec, +} + +async fn scan_snapshots(storage: &OpenDalStorage) -> Result { + let mut result = SnapshotScanResult::default(); + for dir in storage.list_direct_child_dirs().await? { + let manifest_path = format!("{}/{}", dir.trim_matches('/'), MANIFEST_FILE); + let Some(data) = storage.read_file_if_exists(&manifest_path).await? else { + continue; + }; + + match serde_json::from_slice::(&data) { + Ok(manifest) => result.snapshots.push(SnapshotListEntry { + path: format!("{}/", dir.trim_matches('/')), + manifest, + }), + Err(_) => result + .unreadable + .push(format!("{}/", dir.trim_matches('/'))), + } + } + + result + .snapshots + .sort_by_key(|entry| std::cmp::Reverse(entry.manifest.created_at)); + result.unreadable.sort(); + Ok(result) +} + +fn print_snapshot_list(snapshots: &[SnapshotListEntry], unreadable_count: usize) { + if unreadable_count == 0 { + println!("Found {} snapshots:", snapshots.len()); + } else { + println!( + "Found {} snapshots ({} {} skipped: unreadable manifest):", + snapshots.len(), + unreadable_count, + directory_word(unreadable_count) + ); + } + println!(); + println!( + " {:<24} {:<36} {:<19} {:<9} {:<7} {:<6} Status", + "Path", "ID", "Created", "Catalog", "Schemas", "Chunks" + ); + println!( + " {:<24} {:<36} {:<19} {:<9} {:<7} {:<6} {:<10}", + "-".repeat(24), + "-".repeat(36), + "-".repeat(19), + "-".repeat(9), + "-".repeat(7), + "-".repeat(6), + "-".repeat(10) + ); + for entry in snapshots { + let manifest = &entry.manifest; + println!( + " {:<24} {:<36} {:<19} {:<9} {:<7} {:<6} {}", + entry.path, + manifest.snapshot_id, + manifest.created_at.format("%Y-%m-%d %H:%M:%S"), + manifest.catalog, + manifest.schemas.len(), + format_list_chunks(manifest), + snapshot_status(manifest) + ); + } +} + +fn print_unreadable_warnings(unreadable: &[String]) { + if unreadable.is_empty() { + return; + } + + println!(); + println!( + "Warning: {} {} had corrupt/unreadable manifest.json:", + unreadable.len(), + directory_word(unreadable.len()) + ); + for path in unreadable { + println!(" - {}", path); + } +} + +fn directory_word(count: usize) -> &'static str { + if count == 1 { + "directory" + } else { + "directories" + } +} + +fn snapshot_status(manifest: &Manifest) -> &'static str { + if manifest.schema_only { + "schema-only" + } else if manifest.is_complete() { + "complete" + } else { + "incomplete" + } +} + +fn format_list_chunks(manifest: &Manifest) -> String { + let total = manifest.chunks.len(); + if total == 0 { + return "0".to_string(); + } + + format!( + "{}/{}", + manifest.completed_count() + manifest.skipped_count(), + total + ) +} + #[cfg(test)] mod tests { use chrono::TimeZone; use clap::Parser; + use tempfile::tempdir; + use url::Url; use super::*; use crate::data::path::ddl_path_for_schema; @@ -886,4 +1071,124 @@ mod tests { .to_string(); assert!(error.contains("time_range")); } + + #[tokio::test] + async fn test_scan_snapshots_sorts_and_tracks_unreadable_manifests() { + let dir = tempdir().unwrap(); + write_test_manifest( + dir.path(), + "older", + test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + false, + true, + ), + ); + write_test_manifest( + dir.path(), + "newer", + test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 2, 1, 0, 0, 0).unwrap(), + false, + true, + ), + ); + + std::fs::create_dir_all(dir.path().join("empty-dir")).unwrap(); + std::fs::create_dir_all(dir.path().join("not-snapshot")).unwrap(); + std::fs::write(dir.path().join("not-snapshot").join("data.txt"), "x").unwrap(); + std::fs::create_dir_all(dir.path().join("broken")).unwrap(); + std::fs::write(dir.path().join("broken").join(MANIFEST_FILE), "{not-json").unwrap(); + + let uri = Url::from_directory_path(dir.path()).unwrap().to_string(); + let storage = OpenDalStorage::from_file_uri(&uri).unwrap(); + let result = scan_snapshots(&storage).await.unwrap(); + + assert_eq!(result.snapshots.len(), 2); + assert_eq!( + result.snapshots[0].manifest.created_at, + chrono::Utc.with_ymd_and_hms(2026, 2, 1, 0, 0, 0).unwrap() + ); + assert_eq!( + result.snapshots[1].manifest.created_at, + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap() + ); + assert_eq!(result.unreadable, vec!["broken/".to_string()]); + assert_eq!(result.snapshots[0].path, "newer/"); + assert_eq!(result.snapshots[1].path, "older/"); + } + + #[test] + fn test_snapshot_list_status_and_chunk_summary() { + let schema_only = test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + true, + true, + ); + assert_eq!(snapshot_status(&schema_only), "schema-only"); + assert_eq!(format_list_chunks(&schema_only), "0"); + + let complete = test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + false, + true, + ); + assert_eq!(snapshot_status(&complete), "complete"); + assert_eq!(format_list_chunks(&complete), "2/2"); + + let incomplete = test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + false, + false, + ); + assert_eq!(snapshot_status(&incomplete), "incomplete"); + assert_eq!(format_list_chunks(&incomplete), "1/2"); + } + + fn write_test_manifest(root: &std::path::Path, dir: &str, manifest: Manifest) { + let snapshot_dir = root.join(dir); + std::fs::create_dir_all(&snapshot_dir).unwrap(); + std::fs::write( + snapshot_dir.join(MANIFEST_FILE), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + } + + fn test_manifest( + created_at: chrono::DateTime, + schema_only: bool, + complete: bool, + ) -> Manifest { + let mut manifest = Manifest::new_for_export( + "greptime".to_string(), + vec!["public".to_string(), "analytics".to_string()], + schema_only, + TimeRange::unbounded(), + DataFormat::Parquet, + None, + ) + .unwrap(); + manifest.created_at = created_at; + manifest.updated_at = created_at; + + if !schema_only { + manifest.chunks.clear(); + let mut first = ChunkMeta::new(1, TimeRange::unbounded()); + first.mark_completed(vec!["data/public/chunk_1/file.parquet".to_string()], None); + manifest.chunks.push(first); + + if complete { + manifest + .chunks + .push(ChunkMeta::skipped(2, TimeRange::unbounded())); + } else { + manifest + .chunks + .push(ChunkMeta::new(2, TimeRange::unbounded())); + } + } + + manifest + } } diff --git a/src/cli/src/data/snapshot_storage.rs b/src/cli/src/data/snapshot_storage.rs index be94b197df..2ee07b5586 100644 --- a/src/cli/src/data/snapshot_storage.rs +++ b/src/cli/src/data/snapshot_storage.rs @@ -17,6 +17,8 @@ //! This module provides a unified interface for reading and writing snapshot data //! to various storage backends (S3, OSS, GCS, Azure Blob, local filesystem). +use std::collections::BTreeSet; + use async_trait::async_trait; use futures::TryStreamExt; use object_store::services::{Azblob, Fs, Gcs, Oss, S3}; @@ -75,7 +77,10 @@ impl StorageScheme { } /// Extracts bucket/container and root path from a URI. -fn extract_remote_location(uri: &str) -> Result { +fn extract_remote_location_with_root_policy( + uri: &str, + allow_empty_root: bool, +) -> Result { let url = Url::parse(uri).context(UrlParseSnafu)?; let bucket_or_container = url.host_str().unwrap_or("").to_string(); if bucket_or_container.is_empty() { @@ -87,7 +92,7 @@ fn extract_remote_location(uri: &str) -> Result { } let root = url.path().trim_start_matches('/').to_string(); - if root.is_empty() { + if root.is_empty() && !allow_empty_root { return InvalidUriSnafu { uri, reason: "snapshot URI must include a non-empty path after the bucket/container", @@ -268,13 +273,21 @@ impl OpenDalStorage { } fn from_s3_uri(uri: &str, storage: &ObjectStoreConfig) -> Result { + Self::from_s3_uri_with_root_policy(uri, storage, false) + } + + fn from_s3_uri_with_root_policy( + uri: &str, + storage: &ObjectStoreConfig, + allow_empty_root: bool, + ) -> Result { Self::ensure_backend_enabled( uri, storage.enable_s3, "s3:// requires --s3 and related options", )?; - let location = extract_remote_location(uri)?; + let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?; let mut config = storage.s3.clone(); config.s3_bucket = location.bucket_or_container; config.s3_root = location.root; @@ -291,13 +304,21 @@ impl OpenDalStorage { } fn from_oss_uri(uri: &str, storage: &ObjectStoreConfig) -> Result { + Self::from_oss_uri_with_root_policy(uri, storage, false) + } + + fn from_oss_uri_with_root_policy( + uri: &str, + storage: &ObjectStoreConfig, + allow_empty_root: bool, + ) -> Result { Self::ensure_backend_enabled( uri, storage.enable_oss, "oss:// requires --oss and related options", )?; - let location = extract_remote_location(uri)?; + let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?; let mut config = storage.oss.clone(); config.oss_bucket = location.bucket_or_container; config.oss_root = location.root; @@ -314,17 +335,30 @@ impl OpenDalStorage { } fn from_gcs_uri(uri: &str, storage: &ObjectStoreConfig) -> Result { + Self::from_gcs_uri_with_root_policy(uri, storage, false) + } + + fn from_gcs_uri_with_root_policy( + uri: &str, + storage: &ObjectStoreConfig, + allow_empty_root: bool, + ) -> Result { Self::ensure_backend_enabled( uri, storage.enable_gcs, "gs:// or gcs:// requires --gcs and related options", )?; - let location = extract_remote_location(uri)?; + let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?; let mut config = storage.gcs.clone(); config.gcs_bucket = location.bucket_or_container; config.gcs_root = location.root; - Self::validate_remote_config(uri, "gcs", config.validate())?; + // GCS validate() rejects empty root, unlike S3/OSS/Azblob. + if allow_empty_root && config.gcs_root.is_empty() { + Self::validate_gcs_parent_config(uri, &config)?; + } else { + Self::validate_remote_config(uri, "gcs", config.validate())?; + } let conn: GcsConnection = config.into(); let object_store = ObjectStore::new(Gcs::from(&conn)) @@ -336,14 +370,43 @@ impl OpenDalStorage { )) } + fn validate_gcs_parent_config( + uri: &str, + config: &crate::common::PrefixedGcsConnection, + ) -> Result<()> { + if config.gcs_bucket.is_empty() { + return InvalidUriSnafu { + uri, + reason: "invalid gcs config: GCS bucket must be set when --gcs is enabled.", + } + .fail(); + } + if config.gcs_scope.is_empty() { + return InvalidUriSnafu { + uri, + reason: "invalid gcs config: GCS scope must be set when --gcs is enabled.", + } + .fail(); + } + Ok(()) + } + fn from_azblob_uri(uri: &str, storage: &ObjectStoreConfig) -> Result { + Self::from_azblob_uri_with_root_policy(uri, storage, false) + } + + fn from_azblob_uri_with_root_policy( + uri: &str, + storage: &ObjectStoreConfig, + allow_empty_root: bool, + ) -> Result { Self::ensure_backend_enabled( uri, storage.enable_azblob, "azblob:// requires --azblob and related options", )?; - let location = extract_remote_location(uri)?; + let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?; let mut config = storage.azblob.clone(); config.azblob_container = location.bucket_or_container; config.azblob_root = location.root; @@ -370,6 +433,21 @@ impl OpenDalStorage { } } + /// Creates storage rooted at a snapshot parent URI. + /// + /// Parent-oriented commands such as `export-v2 list` may scan bucket/container + /// roots. Snapshot-oriented commands must keep using `from_uri`, which rejects + /// empty remote roots to avoid unsafe snapshot operations at bucket scope. + pub fn from_parent_uri(uri: &str, storage: &ObjectStoreConfig) -> Result { + match StorageScheme::from_uri(uri)? { + StorageScheme::File => Self::from_file_uri_with_config(uri, storage), + StorageScheme::S3 => Self::from_s3_uri_with_root_policy(uri, storage, true), + StorageScheme::Oss => Self::from_oss_uri_with_root_policy(uri, storage, true), + StorageScheme::Gcs => Self::from_gcs_uri_with_root_policy(uri, storage, true), + StorageScheme::Azblob => Self::from_azblob_uri_with_root_policy(uri, storage, true), + } + } + /// Reads a file as bytes. async fn read_file(&self, path: &str) -> Result> { let data = self @@ -382,6 +460,17 @@ impl OpenDalStorage { Ok(data.to_vec()) } + /// Reads a file as bytes if it exists. + pub(crate) async fn read_file_if_exists(&self, path: &str) -> Result>> { + match self.object_store.read(path).await { + Ok(data) => Ok(Some(data.to_vec())), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).context(StorageOperationSnafu { + operation: format!("read {}", path), + }), + } + } + /// Writes bytes to a file. async fn write_file(&self, path: &str, data: Vec) -> Result<()> { self.object_store @@ -404,6 +493,37 @@ impl OpenDalStorage { } } + /// Lists direct child directory names under the storage root. + pub(crate) async fn list_direct_child_dirs(&self) -> Result> { + let mut lister = match self.object_store.lister_with("/").recursive(false).await { + Ok(lister) => lister, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => { + return Err(error).context(StorageOperationSnafu { + operation: "list /", + }); + } + }; + + let mut dirs = BTreeSet::new(); + while let Some(entry) = lister.try_next().await.context(StorageOperationSnafu { + operation: "list /", + })? { + let path = entry.path().trim_matches('/'); + if path.is_empty() { + continue; + } + + if entry.metadata().is_dir() + && let Some(name) = path.split('/').next() + { + dirs.insert(name.to_string()); + } + } + + Ok(dirs.into_iter().collect()) + } + #[cfg(test)] pub async fn read_schema(&self) -> Result { let schemas_path = schema_index_path(); @@ -557,11 +677,35 @@ mod tests { #[test] fn test_extract_remote_location_requires_non_empty_root() { - assert!(extract_remote_location("s3://bucket").is_err()); - assert!(extract_remote_location("s3://bucket/").is_err()); - assert!(extract_remote_location("oss://bucket").is_err()); - assert!(extract_remote_location("gs://bucket").is_err()); - assert!(extract_remote_location("azblob://container").is_err()); + assert!(extract_remote_location_with_root_policy("s3://bucket", false).is_err()); + assert!(extract_remote_location_with_root_policy("s3://bucket/", false).is_err()); + assert!(extract_remote_location_with_root_policy("oss://bucket", false).is_err()); + assert!(extract_remote_location_with_root_policy("gs://bucket", false).is_err()); + assert!(extract_remote_location_with_root_policy("azblob://container", false).is_err()); + } + + #[test] + fn test_extract_remote_location_allows_empty_root_when_permitted() { + let location = extract_remote_location_with_root_policy("s3://bucket", true).unwrap(); + assert_eq!(location.bucket_or_container, "bucket"); + assert_eq!(location.root, ""); + + let location = + extract_remote_location_with_root_policy("azblob://container/", true).unwrap(); + assert_eq!(location.bucket_or_container, "container"); + assert_eq!(location.root, ""); + } + + #[test] + fn test_parent_storage_allows_s3_bucket_root() { + let mut storage = ObjectStoreConfig { + enable_s3: true, + ..Default::default() + }; + storage.s3.s3_region = Some("us-east-1".to_string()); + + assert!(OpenDalStorage::from_uri("s3://bucket", &storage).is_err()); + assert!(OpenDalStorage::from_parent_uri("s3://bucket", &storage).is_ok()); } #[cfg(not(windows))]