feat: implement native histogram read

Signed-off-by: shuiyisong <xixing.sys@gmail.com>
This commit is contained in:
shuiyisong
2026-07-07 10:31:37 +08:00
parent 90752f5649
commit 283f5d91d1
15 changed files with 5299 additions and 104 deletions
Generated
+1
View File
@@ -11021,6 +11021,7 @@ dependencies = [
"bytemuck",
"common-error",
"common-macro",
"common-query",
"common-recordbatch",
"common-telemetry",
"criterion 0.7.0",
+37 -2
View File
@@ -17,6 +17,7 @@ pub mod error;
pub mod logical_plan;
pub mod native_histogram;
pub mod prelude;
pub mod promql_annotations;
pub mod request;
pub mod stream;
#[cfg(any(test, feature = "testing"))]
@@ -32,6 +33,8 @@ use datafusion::physical_plan::ExecutionPlan;
use serde::{Deserialize, Serialize};
use sqlparser_derive::{Visit, VisitMut};
use crate::promql_annotations::PromqlAnnotationCollector;
/// new Output struct with output data(previously Output) and output meta
#[derive(Debug)]
pub struct Output {
@@ -69,6 +72,9 @@ pub struct OutputMeta {
/// May exist for query output. One can retrieve execution metrics from this plan.
pub plan: Option<Arc<dyn ExecutionPlan>>,
pub cost: OutputCost,
pub warnings: Vec<String>,
pub infos: Vec<String>,
pub promql_annotations: Option<PromqlAnnotationCollector>,
}
impl Output {
@@ -121,18 +127,47 @@ impl Debug for OutputData {
impl OutputMeta {
pub fn new(plan: Option<Arc<dyn ExecutionPlan>>, cost: usize) -> Self {
Self { plan, cost }
Self {
plan,
cost,
warnings: Vec::new(),
infos: Vec::new(),
promql_annotations: None,
}
}
pub fn new_with_plan(plan: Arc<dyn ExecutionPlan>) -> Self {
Self {
plan: Some(plan),
cost: 0,
warnings: Vec::new(),
infos: Vec::new(),
promql_annotations: None,
}
}
pub fn new_with_cost(cost: usize) -> Self {
Self { plan: None, cost }
Self {
plan: None,
cost,
warnings: Vec::new(),
infos: Vec::new(),
promql_annotations: None,
}
}
pub fn with_promql_annotations(
mut self,
promql_annotations: Option<PromqlAnnotationCollector>,
) -> Self {
self.promql_annotations = promql_annotations;
self
}
pub fn collect_promql_annotations(&mut self) {
if let Some(collector) = &self.promql_annotations {
collector.append_to(&mut self.warnings, &mut self.infos);
}
}
}
File diff suppressed because it is too large Load Diff
+170
View File
@@ -0,0 +1,170 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::{BTreeSet, HashMap};
use std::sync::{Arc, Mutex, Weak};
use datafusion::config::{ConfigEntry, ConfigExtension, ExtensionOptions};
use datafusion_common::{DataFusionError, Result as DfResult};
use once_cell::sync::Lazy;
type PromqlAnnotationStateRef = Mutex<PromqlAnnotationState>;
static PROMQL_ANNOTATION_REGISTRY: Lazy<Mutex<HashMap<String, Weak<PromqlAnnotationStateRef>>>> =
Lazy::new(|| Mutex::new(HashMap::new()));
#[derive(Debug, Default)]
struct PromqlAnnotationState {
warnings: BTreeSet<String>,
infos: BTreeSet<String>,
}
#[derive(Clone, Debug)]
pub struct PromqlAnnotationCollector {
inner: Arc<PromqlAnnotationStateRef>,
}
impl Default for PromqlAnnotationCollector {
fn default() -> Self {
Self {
inner: Arc::new(Mutex::new(PromqlAnnotationState::default())),
}
}
}
impl PromqlAnnotationCollector {
pub fn record_warning<S: Into<String>>(&self, warning: S) {
self.inner
.lock()
.expect("promql annotation collector poisoned")
.warnings
.insert(warning.into());
}
pub fn record_info<S: Into<String>>(&self, info: S) {
self.inner
.lock()
.expect("promql annotation collector poisoned")
.infos
.insert(info.into());
}
pub fn append_to(&self, warnings: &mut Vec<String>, infos: &mut Vec<String>) {
let inner = self
.inner
.lock()
.expect("promql annotation collector poisoned");
append_unique(warnings, inner.warnings.iter().cloned());
append_unique(infos, inner.infos.iter().cloned());
}
}
impl ConfigExtension for PromqlAnnotationCollector {
const PREFIX: &'static str = "greptime_promql_annotations";
}
impl ExtensionOptions for PromqlAnnotationCollector {
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
fn cloned(&self) -> Box<dyn ExtensionOptions> {
Box::new(self.clone())
}
fn set(&mut self, key: &str, value: &str) -> DfResult<()> {
Err(DataFusionError::NotImplemented(format!(
"PromqlAnnotationCollector does not support set key: {key} with value: {value}"
)))
}
fn entries(&self) -> Vec<ConfigEntry> {
vec![]
}
}
pub fn promql_annotation_collector(query_id: &str) -> PromqlAnnotationCollector {
let mut registry = PROMQL_ANNOTATION_REGISTRY
.lock()
.expect("promql annotation registry poisoned");
registry.retain(|_, collector| collector.strong_count() > 0);
if let Some(inner) = registry.get(query_id).and_then(Weak::upgrade) {
return PromqlAnnotationCollector { inner };
}
let collector = PromqlAnnotationCollector::default();
registry.insert(query_id.to_string(), Arc::downgrade(&collector.inner));
collector
}
pub fn get_promql_annotation_collector(query_id: &str) -> Option<PromqlAnnotationCollector> {
let mut registry = PROMQL_ANNOTATION_REGISTRY
.lock()
.expect("promql annotation registry poisoned");
registry.retain(|_, collector| collector.strong_count() > 0);
registry
.get(query_id)
.and_then(Weak::upgrade)
.map(|inner| PromqlAnnotationCollector { inner })
}
fn append_unique(target: &mut Vec<String>, values: impl Iterator<Item = String>) {
let mut seen = target.iter().cloned().collect::<BTreeSet<_>>();
for value in values {
if seen.insert(value.clone()) {
target.push(value);
}
}
target.sort();
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn collector_deduplicates_and_orders_annotations() {
let collector = PromqlAnnotationCollector::default();
collector.record_info("z-info");
collector.record_info("a-info");
collector.record_info("z-info");
collector.record_warning("b-warning");
collector.record_warning("a-warning");
let mut warnings = vec!["c-warning".to_string()];
let mut infos = vec![];
collector.append_to(&mut warnings, &mut infos);
assert_eq!(warnings, vec!["a-warning", "b-warning", "c-warning"]);
assert_eq!(infos, vec!["a-info", "z-info"]);
}
#[test]
fn registry_prunes_dead_collectors() {
let key = "registry_prunes_dead_collectors";
{
let collector = promql_annotation_collector(key);
assert!(get_promql_annotation_collector(key).is_some());
drop(collector);
}
assert!(get_promql_annotation_collector(key).is_none());
assert!(!PROMQL_ANNOTATION_REGISTRY.lock().unwrap().contains_key(key));
}
}
+1
View File
@@ -13,6 +13,7 @@ async-trait.workspace = true
bytemuck.workspace = true
common-error.workspace = true
common-macro.workspace = true
common-query.workspace = true
common-telemetry.workspace = true
datafusion.workspace = true
datafusion-common.workspace = true
+13
View File
@@ -18,6 +18,7 @@ mod deriv;
mod double_exponential_smoothing;
mod extrapolate_rate;
mod idelta;
mod native_histogram;
mod predict_linear;
mod quantile;
mod quantile_aggr;
@@ -42,6 +43,18 @@ pub use deriv::Deriv;
pub use double_exponential_smoothing::DoubleExponentialSmoothing;
pub use extrapolate_rate::{Delta, Increase, Rate};
pub use idelta::IDelta;
pub use native_histogram::{
NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg,
NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges,
NativeHistogramCount, NativeHistogramCountOverTime, NativeHistogramDelta,
NativeHistogramDivScalar, NativeHistogramDrop, NativeHistogramEq, NativeHistogramFraction,
NativeHistogramIDelta, NativeHistogramIRate, NativeHistogramIncrease,
NativeHistogramLastOverTime, NativeHistogramMulScalar, NativeHistogramNeg,
NativeHistogramNotEq, NativeHistogramPresentOverTime, NativeHistogramQuantile,
NativeHistogramRate, NativeHistogramResets, NativeHistogramScalarMul, NativeHistogramStddev,
NativeHistogramStdvar, NativeHistogramSub, NativeHistogramSum, NativeHistogramSumOverTime,
NativeHistogramToString,
};
pub use predict_linear::PredictLinear;
pub use quantile::QuantileOverTime;
pub use quantile_aggr::{QUANTILE_NAME, quantile_udaf};
File diff suppressed because it is too large Load Diff
+17 -2
View File
@@ -28,6 +28,7 @@ use common_catalog::consts::is_readonly_schema;
use common_error::ext::BoxedError;
use common_function::function::FunctionContext;
use common_function::function_factory::ScalarFunctionFactory;
use common_query::promql_annotations::get_promql_annotation_collector;
use common_query::{Output, OutputData, OutputMeta};
use common_recordbatch::adapter::{RecordBatchStreamAdapter, RegionQueryStatCounters};
use common_recordbatch::{EmptyRecordBatchStream, SendableRecordBatchStream};
@@ -167,16 +168,19 @@ impl DatafusionQueryEngine {
let physical_plan = self.create_physical_plan(&mut ctx, &plan).await?;
let physical_plan = self.optimize_physical_plan(&mut ctx, physical_plan)?;
let physical_plan = if let Some(wrapper) = self.plugins.get::<PhysicalPlanWrapperRef>() {
wrapper.wrap(physical_plan, query_ctx)
wrapper.wrap(physical_plan, query_ctx.clone())
} else {
physical_plan
};
let stream = self.execute_stream(&ctx, &physical_plan)?;
let promql_annotations = query_ctx
.remote_query_id()
.and_then(get_promql_annotation_collector);
Ok(Output::new(
OutputData::Stream(stream),
OutputMeta::new_with_plan(physical_plan),
OutputMeta::new_with_plan(physical_plan).with_promql_annotations(promql_annotations),
))
}
@@ -597,6 +601,17 @@ impl QueryEngine for DatafusionQueryEngine {
state: self.engine_state().function_state(),
});
if let Some(promql_annotations) = query_ctx
.remote_query_id()
.and_then(get_promql_annotation_collector)
{
state
.config_mut()
.options_mut()
.extensions
.insert(promql_annotations);
}
// Carry scheduled Flow time through ConfigOptions.extensions so that
// the distributed plan analyzer can read it during expression
// simplification (preventing wall-clock constant-folding of `now()`).
+19 -5
View File
@@ -22,6 +22,7 @@ use arrow_schema::DataType;
use async_trait::async_trait;
use catalog::table_source::DfTableSourceProvider;
use common_error::ext::BoxedError;
use common_query::promql_annotations::promql_annotation_collector;
use common_telemetry::tracing;
use datafusion::common::{DFSchema, plan_err};
use datafusion::execution::SessionStateBuilder;
@@ -299,7 +300,15 @@ impl DfLogicalPlanner {
#[tracing::instrument(skip_all)]
async fn plan_pql(&self, stmt: &EvalStmt, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
let scheduled_state = self.derive_session_state_with_scheduled_time(&query_ctx)?;
let mut scheduled_state = self.derive_session_state_with_scheduled_time(&query_ctx)?;
let promql_annotations = query_ctx.remote_query_id().map(promql_annotation_collector);
if let Some(collector) = &promql_annotations {
scheduled_state
.config_mut()
.options_mut()
.extensions
.insert(collector.clone());
}
let plan_decoder = Arc::new(DefaultPlanDecoder::new(
scheduled_state.clone(),
&query_ctx,
@@ -314,10 +323,15 @@ impl DfLogicalPlanner {
.sql_parser
.enable_ident_normalization,
);
let plan = PromPlanner::stmt_to_plan(table_provider, stmt, &self.engine_state)
.await
.map_err(BoxedError::new)
.context(QueryPlanSnafu)?;
let plan = PromPlanner::stmt_to_plan_with_annotations(
table_provider,
stmt,
&self.engine_state,
promql_annotations,
)
.await
.map_err(BoxedError::new)
.context(QueryPlanSnafu)?;
let context = QueryEngineContext::new(scheduled_state, query_ctx);
Ok(self
+665 -26
View File
@@ -22,7 +22,9 @@ use catalog::table_source::DfTableSourceProvider;
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_function::function::FunctionContext;
use common_query::native_histogram::native_histogram_value_type;
use common_query::prelude::greptime_value;
use common_query::promql_annotations::PromqlAnnotationCollector;
use datafusion::common::DFSchemaRef;
use datafusion::datasource::DefaultTableSource;
use datafusion::functions_aggregate::average::avg_udaf;
@@ -51,7 +53,7 @@ use datafusion_expr::{
ExprSchemable, Literal, Projection, SortExpr, TableScan, TableSource, col, lit,
};
use datatypes::arrow::datatypes::{DataType as ArrowDataType, TimeUnit as ArrowTimeUnit};
use datatypes::data_type::ConcreteDataType;
use datatypes::data_type::{ConcreteDataType, DataType as GreptimeDataType};
use itertools::Itertools;
use once_cell::sync::Lazy;
use promql::extension_plan::{
@@ -60,9 +62,18 @@ use promql::extension_plan::{
};
use promql::functions::{
AbsentOverTime, AvgOverTime, Changes, CountOverTime, Delta, Deriv, DoubleExponentialSmoothing,
IDelta, Increase, LastOverTime, MaxOverTime, MinOverTime, PredictLinear, PresentOverTime,
QuantileOverTime, Rate, Resets, Round, StddevOverTime, StdvarOverTime, SumOverTime,
quantile_udaf,
IDelta, Increase, LastOverTime, MaxOverTime, MinOverTime, NativeHistogramAbsentOverTime,
NativeHistogramAdd, NativeHistogramAggAvg, NativeHistogramAggSum, NativeHistogramAvg,
NativeHistogramAvgOverTime, NativeHistogramChanges, NativeHistogramCount,
NativeHistogramCountOverTime, NativeHistogramDelta, NativeHistogramDivScalar,
NativeHistogramDrop, NativeHistogramEq, NativeHistogramFraction, NativeHistogramIDelta,
NativeHistogramIRate, NativeHistogramIncrease, NativeHistogramLastOverTime,
NativeHistogramMulScalar, NativeHistogramNeg, NativeHistogramNotEq,
NativeHistogramPresentOverTime, NativeHistogramQuantile, NativeHistogramRate,
NativeHistogramResets, NativeHistogramScalarMul, NativeHistogramStddev, NativeHistogramStdvar,
NativeHistogramSub, NativeHistogramSum, NativeHistogramSumOverTime, NativeHistogramToString,
PredictLinear, PresentOverTime, QuantileOverTime, Rate, Resets, Round, StddevOverTime,
StdvarOverTime, SumOverTime, quantile_udaf,
};
use promql_parser::label::{METRIC_NAME, MatchOp, Matcher, Matchers};
use promql_parser::parser::token::TokenType;
@@ -374,6 +385,7 @@ impl PromPlannerContext {
pub struct PromPlanner {
table_provider: DfTableSourceProvider,
ctx: PromPlannerContext,
promql_annotations: Option<PromqlAnnotationCollector>,
}
impl PromPlanner {
@@ -381,10 +393,20 @@ impl PromPlanner {
table_provider: DfTableSourceProvider,
stmt: &EvalStmt,
query_engine_state: &QueryEngineState,
) -> Result<LogicalPlan> {
Self::stmt_to_plan_with_annotations(table_provider, stmt, query_engine_state, None).await
}
pub async fn stmt_to_plan_with_annotations(
table_provider: DfTableSourceProvider,
stmt: &EvalStmt,
query_engine_state: &QueryEngineState,
promql_annotations: Option<PromqlAnnotationCollector>,
) -> Result<LogicalPlan> {
let mut planner = Self {
table_provider,
ctx: PromPlannerContext::from_eval_stmt(stmt),
promql_annotations,
};
let plan = planner
@@ -699,6 +721,22 @@ impl PromPlanner {
let group_exprs = self.agg_modifier_to_col(input.schema(), modifier, false)?;
if self.all_field_columns_are_native_histograms(input.schema()) {
let promql_annotations = self.promql_annotations.clone();
return self.projection_for_each_field_column(input, |col| {
Ok(DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(NativeHistogramDrop::float_null_udf(
format!(
"{}: dropped native histogram samples because this aggregation is not supported for native histograms",
op
),
promql_annotations.clone(),
)),
args: vec![DfExpr::Column(Column::from_name(col))],
}))
});
}
let val = Self::get_param_as_literal_expr(param, Some(*op), Some(ArrowDataType::Float64))?;
// convert op and value columns to window exprs.
@@ -775,6 +813,14 @@ impl PromPlanner {
let UnaryExpr { expr } = unary_expr;
// Unary Expr in PromQL implys the `-` operator
let input = self.prom_expr_to_plan(expr, query_engine_state).await?;
if self.all_field_columns_are_native_histograms(input.schema()) {
return self.projection_for_each_field_column(input, |col| {
Ok(DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(NativeHistogramNeg::scalar_udf()),
args: vec![DfExpr::Column(col.into())],
}))
});
}
self.projection_for_each_field_column(input, |col| {
Ok(DfExpr::Negative(Box::new(DfExpr::Column(col.into()))))
})
@@ -818,6 +864,16 @@ impl PromPlanner {
});
}
if planned_leaves.iter().any(|leaf| {
leaf.ctx
.field_columns
.iter()
.any(|field| Self::field_column_is_native_histogram(leaf.plan.schema(), field))
}) {
self.ctx = original_ctx;
return Ok(None);
}
if !Self::binary_island_join_contexts_supported(&planned_leaves) {
self.ctx = original_ctx;
return Ok(None);
@@ -1154,10 +1210,25 @@ impl PromPlanner {
if let Some(time_expr) = self.try_build_special_time_expr_with_context(lhs) {
expr = time_expr
}
let input_schema = input.schema().clone();
let promql_annotations = self.promql_annotations.clone();
let bin_expr_builder = |col: &String| {
let binary_expr_builder = Self::prom_token_to_binary_expr_builder(*op)?;
let mut binary_expr =
binary_expr_builder(expr.clone(), DfExpr::Column(col.into()))?;
let rhs_is_histogram =
Self::field_column_is_native_histogram(&input_schema, col);
let rhs = DfExpr::Column(col.into());
let mut binary_expr = match Self::native_histogram_binary_expr(
*op,
expr.clone(),
false,
rhs.clone(),
rhs_is_histogram,
is_comparison_op && !should_return_bool,
promql_annotations.clone(),
)? {
Some(expr) => expr,
None => binary_expr_builder(expr.clone(), rhs)?,
};
if is_comparison_op && should_return_bool {
binary_expr = DfExpr::Cast(Cast {
@@ -1180,10 +1251,25 @@ impl PromPlanner {
if let Some(time_expr) = self.try_build_special_time_expr_with_context(rhs) {
expr = time_expr
}
let input_schema = input.schema().clone();
let promql_annotations = self.promql_annotations.clone();
let bin_expr_builder = |col: &String| {
let binary_expr_builder = Self::prom_token_to_binary_expr_builder(*op)?;
let mut binary_expr =
binary_expr_builder(DfExpr::Column(col.into()), expr.clone())?;
let lhs_is_histogram =
Self::field_column_is_native_histogram(&input_schema, col);
let lhs = DfExpr::Column(col.into());
let mut binary_expr = match Self::native_histogram_binary_expr(
*op,
lhs.clone(),
lhs_is_histogram,
expr.clone(),
false,
is_comparison_op && !should_return_bool,
promql_annotations.clone(),
)? {
Some(expr) => expr,
None => binary_expr_builder(lhs, expr.clone())?,
};
if is_comparison_op && should_return_bool {
binary_expr = DfExpr::Cast(Cast {
@@ -1279,21 +1365,38 @@ impl PromPlanner {
&right_context,
)?;
let join_plan_schema = join_plan.schema().clone();
let promql_annotations = self.promql_annotations.clone();
let bin_expr_builder = |_: &String| {
let (left_col_name, right_col_name) = field_columns.next().unwrap();
let left_col = join_plan_schema
let left_field = join_plan_schema
.qualified_field_with_name(Some(&left_table_ref), left_col_name)
.context(DataFusionPlanningSnafu)?
.into();
let right_col = join_plan_schema
.context(DataFusionPlanningSnafu)?;
let right_field = join_plan_schema
.qualified_field_with_name(Some(&right_table_ref), right_col_name)
.context(DataFusionPlanningSnafu)?
.into();
.context(DataFusionPlanningSnafu)?;
let left_is_histogram =
left_field.1.data_type() == &Self::native_histogram_arrow_type();
let right_is_histogram =
right_field.1.data_type() == &Self::native_histogram_arrow_type();
let left_col = left_field.into();
let right_col = right_field.into();
let binary_expr_builder = Self::prom_token_to_binary_expr_builder(*op)?;
let mut binary_expr =
binary_expr_builder(DfExpr::Column(left_col), DfExpr::Column(right_col))?;
let lhs = DfExpr::Column(left_col);
let rhs = DfExpr::Column(right_col);
let mut binary_expr = match Self::native_histogram_binary_expr(
*op,
lhs.clone(),
left_is_histogram,
rhs.clone(),
right_is_histogram,
is_comparison_op && !should_return_bool,
promql_annotations.clone(),
)? {
Some(expr) => expr,
None => binary_expr_builder(lhs, rhs)?,
};
if is_comparison_op && should_return_bool {
binary_expr = DfExpr::Cast(Cast {
expr: Box::new(binary_expr),
@@ -1613,8 +1716,12 @@ impl PromPlanner {
),
})
};
let (mut func_exprs, new_tags) =
self.create_function_expr(func, args.literals.clone(), query_engine_state)?;
let (mut func_exprs, new_tags) = self.create_function_expr(
func,
args.literals.clone(),
input.schema(),
query_engine_state,
)?;
func_exprs.insert(0, self.create_time_index_column_expr()?);
func_exprs.extend_from_slice(&self.create_tag_column_exprs()?);
if let Some(tsid_col) =
@@ -2738,6 +2845,7 @@ impl PromPlanner {
&mut self,
func: &Function,
other_input_exprs: Vec<DfExpr>,
input_schema: &DFSchemaRef,
query_engine_state: &QueryEngineState,
) -> Result<(Vec<DfExpr>, Vec<String>)> {
// TODO(ruihang): check function args list
@@ -2748,35 +2856,136 @@ impl PromPlanner {
let mut exprs = Vec::with_capacity(self.ctx.field_columns.len());
// New labels after executing the function, e.g. `label_replace` etc.
let mut new_tags = vec![];
let promql_annotations = self.promql_annotations.clone();
let native_histogram_drop_udf = |name: &str| {
Arc::new(NativeHistogramDrop::float_null_udf(
format!(
"{name}: dropped native histogram samples because this function is not supported for native histograms"
),
promql_annotations.clone(),
))
};
let scalar_func = match func.name {
"increase" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::ExtrapolateUdf(
Arc::new(NativeHistogramIncrease::scalar_udf_with_collector(
self.promql_annotations.clone(),
)),
self.ctx.range.context(ExpectRangeSelectorSnafu)?,
)
}
"increase" => ScalarFunc::ExtrapolateUdf(
Arc::new(Increase::scalar_udf()),
self.ctx.range.context(ExpectRangeSelectorSnafu)?,
),
"rate" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::ExtrapolateUdf(
Arc::new(NativeHistogramRate::scalar_udf_with_collector(
self.promql_annotations.clone(),
)),
self.ctx.range.context(ExpectRangeSelectorSnafu)?,
)
}
"rate" => ScalarFunc::ExtrapolateUdf(
Arc::new(Rate::scalar_udf()),
self.ctx.range.context(ExpectRangeSelectorSnafu)?,
),
"delta" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::ExtrapolateUdf(
Arc::new(NativeHistogramDelta::scalar_udf_with_collector(
self.promql_annotations.clone(),
)),
self.ctx.range.context(ExpectRangeSelectorSnafu)?,
)
}
"delta" => ScalarFunc::ExtrapolateUdf(
Arc::new(Delta::scalar_udf()),
self.ctx.range.context(ExpectRangeSelectorSnafu)?,
),
"idelta" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(Arc::new(NativeHistogramIDelta::scalar_udf_with_collector(
self.promql_annotations.clone(),
)))
}
"idelta" => ScalarFunc::Udf(Arc::new(IDelta::<false>::scalar_udf())),
"irate" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(Arc::new(NativeHistogramIRate::scalar_udf_with_collector(
self.promql_annotations.clone(),
)))
}
"irate" => ScalarFunc::Udf(Arc::new(IDelta::<true>::scalar_udf())),
"resets" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(Arc::new(NativeHistogramResets::scalar_udf()))
}
"resets" => ScalarFunc::Udf(Arc::new(Resets::scalar_udf())),
"changes" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(Arc::new(NativeHistogramChanges::scalar_udf()))
}
"changes" => ScalarFunc::Udf(Arc::new(Changes::scalar_udf())),
"deriv" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(native_histogram_drop_udf(func.name))
}
"deriv" => ScalarFunc::Udf(Arc::new(Deriv::scalar_udf())),
"avg_over_time" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(Arc::new(
NativeHistogramAvgOverTime::scalar_udf_with_collector(
self.promql_annotations.clone(),
),
))
}
"avg_over_time" => ScalarFunc::Udf(Arc::new(AvgOverTime::scalar_udf())),
"min_over_time" | "max_over_time"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(native_histogram_drop_udf(func.name))
}
"min_over_time" => ScalarFunc::Udf(Arc::new(MinOverTime::scalar_udf())),
"max_over_time" => ScalarFunc::Udf(Arc::new(MaxOverTime::scalar_udf())),
"sum_over_time" if self.all_field_columns_are_native_histogram_ranges(input_schema) => {
ScalarFunc::Udf(Arc::new(
NativeHistogramSumOverTime::scalar_udf_with_collector(
self.promql_annotations.clone(),
),
))
}
"sum_over_time" => ScalarFunc::Udf(Arc::new(SumOverTime::scalar_udf())),
"count_over_time"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(Arc::new(NativeHistogramCountOverTime::scalar_udf()))
}
"count_over_time" => ScalarFunc::Udf(Arc::new(CountOverTime::scalar_udf())),
"last_over_time"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(Arc::new(NativeHistogramLastOverTime::scalar_udf()))
}
"last_over_time" => ScalarFunc::Udf(Arc::new(LastOverTime::scalar_udf())),
"absent_over_time"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(Arc::new(NativeHistogramAbsentOverTime::scalar_udf()))
}
"absent_over_time" => ScalarFunc::Udf(Arc::new(AbsentOverTime::scalar_udf())),
"present_over_time"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(Arc::new(NativeHistogramPresentOverTime::scalar_udf()))
}
"present_over_time" => ScalarFunc::Udf(Arc::new(PresentOverTime::scalar_udf())),
"stddev_over_time" | "stdvar_over_time" | "quantile_over_time"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(native_histogram_drop_udf(func.name))
}
"stddev_over_time" => ScalarFunc::Udf(Arc::new(StddevOverTime::scalar_udf())),
"stdvar_over_time" => ScalarFunc::Udf(Arc::new(StdvarOverTime::scalar_udf())),
"quantile_over_time" => ScalarFunc::Udf(Arc::new(QuantileOverTime::scalar_udf())),
"predict_linear"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(native_histogram_drop_udf(func.name))
}
"predict_linear" => {
other_input_exprs[0] = DfExpr::Cast(Cast {
expr: Box::new(other_input_exprs[0].clone()),
@@ -2784,9 +2993,32 @@ impl PromPlanner {
});
ScalarFunc::Udf(Arc::new(PredictLinear::scalar_udf()))
}
"double_exponential_smoothing" | "holt_winters"
if self.all_field_columns_are_native_histogram_ranges(input_schema) =>
{
ScalarFunc::Udf(native_histogram_drop_udf(func.name))
}
"double_exponential_smoothing" | "holt_winters" => {
ScalarFunc::Udf(Arc::new(DoubleExponentialSmoothing::scalar_udf()))
}
"histogram_count" => {
ScalarFunc::NativeHistogramUdf(Arc::new(NativeHistogramCount::scalar_udf()))
}
"histogram_sum" => {
ScalarFunc::NativeHistogramUdf(Arc::new(NativeHistogramSum::scalar_udf()))
}
"histogram_avg" => {
ScalarFunc::NativeHistogramUdf(Arc::new(NativeHistogramAvg::scalar_udf()))
}
"histogram_stddev" => {
ScalarFunc::NativeHistogramUdf(Arc::new(NativeHistogramStddev::scalar_udf()))
}
"histogram_stdvar" => {
ScalarFunc::NativeHistogramUdf(Arc::new(NativeHistogramStdvar::scalar_udf()))
}
"histogram_fraction" => {
ScalarFunc::NativeHistogramUdf(Arc::new(NativeHistogramFraction::scalar_udf()))
}
"time" => {
exprs.push(build_special_time_expr(
self.ctx.time_index_column.as_ref().unwrap(),
@@ -2935,12 +3167,21 @@ impl PromPlanner {
ScalarFunc::GeneratedExpr
}
"round" if self.all_field_columns_are_native_histograms(input_schema) => {
if other_input_exprs.is_empty() {
other_input_exprs.push_front(0.0f64.lit());
}
ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name))
}
"round" => {
if other_input_exprs.is_empty() {
other_input_exprs.push_front(0.0f64.lit());
}
ScalarFunc::DataFusionUdf(Arc::new(Round::scalar_udf()))
}
"rad" | "deg" | "sgn" if self.all_field_columns_are_native_histograms(input_schema) => {
ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name))
}
"rad" => ScalarFunc::DataFusionBuiltin(datafusion::functions::math::radians()),
"deg" => ScalarFunc::DataFusionBuiltin(datafusion::functions::math::degrees()),
"sgn" => ScalarFunc::DataFusionBuiltin(datafusion::functions::math::signum()),
@@ -2960,20 +3201,32 @@ impl PromPlanner {
.scalar_functions()
.get(func.name)
{
ScalarFunc::DataFusionBuiltin(f.clone())
if self.all_field_columns_are_native_histograms(input_schema) {
ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name))
} else {
ScalarFunc::DataFusionBuiltin(f.clone())
}
} else if let Some(factory) = query_engine_state.scalar_function(func.name) {
let func_state = query_engine_state.function_state();
let query_ctx = self.table_provider.query_ctx();
if self.all_field_columns_are_native_histograms(input_schema) {
ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name))
} else {
let func_state = query_engine_state.function_state();
let query_ctx = self.table_provider.query_ctx();
ScalarFunc::DataFusionUdf(Arc::new(factory.provide(FunctionContext {
state: func_state,
query_ctx: query_ctx.clone(),
})))
ScalarFunc::DataFusionUdf(Arc::new(factory.provide(FunctionContext {
state: func_state,
query_ctx: query_ctx.clone(),
})))
}
} else if let Some(f) = datafusion_functions::math::functions()
.iter()
.find(|f| f.name() == func.name)
{
ScalarFunc::DataFusionUdf(f.clone())
if self.all_field_columns_are_native_histograms(input_schema) {
ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name))
} else {
ScalarFunc::DataFusionUdf(f.clone())
}
} else {
return UnsupportedExprSnafu {
name: func.name.to_string(),
@@ -3005,6 +3258,25 @@ impl PromPlanner {
.collect_vec();
exprs.push(DfExpr::ScalarFunction(ScalarFunction { func, args }))
}
ScalarFunc::NativeHistogramUdf(func) => {
if Self::field_column_is_native_histogram(input_schema, value) {
let args = itertools::chain!(
other_input_exprs.iter().take(field_column_pos).cloned(),
std::iter::once(col_expr),
other_input_exprs.iter().skip(field_column_pos).cloned()
)
.collect_vec();
exprs.push(DfExpr::ScalarFunction(ScalarFunction { func, args }));
} else {
exprs.push(
DfExpr::Literal(ScalarValue::Float64(None), None).alias(format!(
"{}_{}",
func.name(),
value
)),
);
}
}
ScalarFunc::Udf(func) => {
let ts_range_expr = DfExpr::Column(Column::from_name(
RangeManipulate::build_timestamp_range_name(
@@ -3391,6 +3663,11 @@ impl PromPlanner {
}
);
}
if self.all_field_columns_are_native_histograms(input_plan.schema()) {
return self.create_native_histogram_aggregate_exprs(op, input_plan);
}
let aggr = match op.id() {
token::T_SUM => sum_udaf(),
token::T_QUANTILE => {
@@ -3466,6 +3743,85 @@ impl PromPlanner {
Ok((exprs, prev_field_exprs))
}
fn create_native_histogram_aggregate_exprs(
&mut self,
op: TokenType,
input_plan: &LogicalPlan,
) -> Result<(Vec<DfExpr>, Vec<DfExpr>)> {
let prev_field_exprs = if op.id() == token::T_COUNT_VALUES {
ensure!(
self.ctx.field_columns.len() == 1,
UnsupportedExprSnafu {
name: "count_values on multi-value input"
}
);
self.ctx
.field_columns
.iter()
.map(|col| {
DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(NativeHistogramToString::scalar_udf()),
args: vec![DfExpr::Column(Column::from_name(col))],
})
})
.collect::<Vec<_>>()
} else {
vec![]
};
let exprs = self
.ctx
.field_columns
.iter()
.map(|col| {
let input = DfExpr::Column(Column::from_name(col));
let expr = match op.id() {
token::T_SUM => Arc::new(NativeHistogramAggSum::aggregate_udf_with_collector(
self.promql_annotations.clone(),
))
.call(vec![input])
.alias(col),
token::T_AVG => Arc::new(NativeHistogramAggAvg::aggregate_udf_with_collector(
self.promql_annotations.clone(),
))
.call(vec![input])
.alias(col),
token::T_COUNT_VALUES | token::T_COUNT => count_udaf().call(vec![input]).alias(col),
token::T_GROUP => max_udaf().call(vec![lit(1_f64)]).alias(col),
token::T_MIN
| token::T_MAX
| token::T_STDDEV
| token::T_STDVAR
| token::T_QUANTILE
| token::T_TOPK
| token::T_BOTTOMK => sum_udaf()
.call(vec![DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(NativeHistogramDrop::float_null_udf(
format!(
"{}: dropped native histogram samples because this aggregation is not supported for native histograms",
op
),
self.promql_annotations.clone(),
)),
args: vec![input],
})])
.alias(col),
_ => return UnexpectedTokenSnafu { token: op }.fail(),
};
Ok(expr)
})
.collect::<Result<Vec<_>>>()?;
let normalized_exprs =
normalize_cols(exprs.iter().cloned(), input_plan).context(DataFusionPlanningSnafu)?;
self.ctx.field_columns = normalized_exprs
.into_iter()
.map(|expr| expr.schema_name().to_string())
.collect();
Ok((exprs, prev_field_exprs))
}
fn get_param_value_as_str(op: TokenType, param: &Option<Box<PromExpr>>) -> Result<&str> {
let param = param
.as_deref()
@@ -3634,6 +3990,10 @@ impl PromPlanner {
let input_plan = self.strip_tsid_column(input_plan)?;
self.ctx.use_tsid = false;
if self.all_field_columns_are_native_histograms(input_plan.schema()) {
return self.create_native_histogram_quantile_plan(phi, input_plan);
}
if !self.ctx.has_le_tag() {
// Return empty result instead of error when 'le' column is not found
// This handles the case when histogram metrics don't exist
@@ -3677,6 +4037,43 @@ impl PromPlanner {
}))
}
fn create_native_histogram_quantile_plan(
&mut self,
phi: f64,
input_plan: LogicalPlan,
) -> Result<LogicalPlan> {
ensure!(
self.ctx.field_columns.len() == 1,
MultiFieldsNotSupportedSnafu {
operator: SPECIAL_HISTOGRAM_QUANTILE
},
);
let field_column = self.ctx.field_columns[0].clone();
let quantile_expr = DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(NativeHistogramQuantile::scalar_udf()),
args: vec![
DfExpr::Column(Column::from_name(field_column)),
DfExpr::Literal(ScalarValue::Float64(Some(phi)), None),
],
});
let display_name = quantile_expr.schema_name().to_string();
self.ctx.field_columns = vec![display_name.clone()];
let project_exprs = std::iter::once(self.create_time_index_column_expr()?)
.chain(std::iter::once(quantile_expr.alias(display_name)))
.chain(self.create_tag_column_exprs()?)
.collect::<Vec<_>>();
LogicalPlanBuilder::from(input_plan)
.project(project_exprs)
.context(DataFusionPlanningSnafu)?
.filter(self.create_empty_values_filter_expr()?)
.context(DataFusionPlanningSnafu)?
.build()
.context(DataFusionPlanningSnafu)
}
/// Create a [SPECIAL_VECTOR_FUNCTION] plan
async fn create_vector_plan(&mut self, args: &PromFunctionArgs) -> Result<LogicalPlan> {
if args.args.len() != 1 {
@@ -3884,6 +4281,69 @@ impl PromPlanner {
}
}
fn native_histogram_binary_expr(
token: TokenType,
lhs: DfExpr,
lhs_is_histogram: bool,
rhs: DfExpr,
rhs_is_histogram: bool,
filter_context: bool,
promql_annotations: Option<PromqlAnnotationCollector>,
) -> Result<Option<DfExpr>> {
if !lhs_is_histogram && !rhs_is_histogram {
return Ok(None);
}
let scalar_fn = |func: ScalarUdfDef, args| {
DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(func),
args,
})
};
let invalid_expr = || {
let message = format!(
"{}: dropped native histogram samples because this binary operation is not supported for native histograms",
token
);
let func = if filter_context {
NativeHistogramDrop::bool_false_udf(message, promql_annotations.clone())
} else {
NativeHistogramDrop::float_null_udf(message, promql_annotations.clone())
};
let args = vec![lhs.clone(), rhs.clone()];
scalar_fn(func, args)
};
let expr = match (token.id(), lhs_is_histogram, rhs_is_histogram) {
(token::T_ADD, true, true) => scalar_fn(
NativeHistogramAdd::scalar_udf_with_collector(promql_annotations.clone()),
vec![lhs, rhs],
),
(token::T_SUB, true, true) => scalar_fn(
NativeHistogramSub::scalar_udf_with_collector(promql_annotations.clone()),
vec![lhs, rhs],
),
(token::T_MUL, true, false) => {
scalar_fn(NativeHistogramMulScalar::scalar_udf(), vec![lhs, rhs])
}
(token::T_MUL, false, true) => {
scalar_fn(NativeHistogramScalarMul::scalar_udf(), vec![lhs, rhs])
}
(token::T_DIV, true, false) => {
scalar_fn(NativeHistogramDivScalar::scalar_udf(), vec![lhs, rhs])
}
(token::T_EQLC, true, true) => {
scalar_fn(NativeHistogramEq::scalar_udf(), vec![lhs, rhs])
}
(token::T_NEQ, true, true) => {
scalar_fn(NativeHistogramNotEq::scalar_udf(), vec![lhs, rhs])
}
_ => invalid_expr(),
};
Ok(Some(expr))
}
/// Return a lambda to build binary expression from token.
/// Because some binary operator are function in DataFusion like `atan2` or `^`.
#[allow(clippy::type_complexity)]
@@ -3990,6 +4450,52 @@ impl PromPlanner {
.any(|field| field.name() == DATA_SCHEMA_TSID_COLUMN_NAME)
}
fn native_histogram_arrow_type() -> ArrowDataType {
native_histogram_value_type().as_arrow_type()
}
fn field_column_type<'a>(
schema: &'a DFSchemaRef,
field_column: &str,
) -> Option<&'a ArrowDataType> {
schema
.index_of_column_by_name(None, field_column)
.map(|idx| schema.field(idx).data_type())
}
fn field_column_is_native_histogram(schema: &DFSchemaRef, field_column: &str) -> bool {
Self::field_column_type(schema, field_column)
.is_some_and(|data_type| data_type == &Self::native_histogram_arrow_type())
}
fn field_column_is_native_histogram_range(schema: &DFSchemaRef, field_column: &str) -> bool {
Self::field_column_type(schema, field_column).is_some_and(|data_type| {
matches!(
data_type,
ArrowDataType::Dictionary(_, value_type)
if value_type.as_ref() == &Self::native_histogram_arrow_type()
)
})
}
fn all_field_columns_are_native_histograms(&self, schema: &DFSchemaRef) -> bool {
!self.ctx.field_columns.is_empty()
&& self
.ctx
.field_columns
.iter()
.all(|field| Self::field_column_is_native_histogram(schema, field))
}
fn all_field_columns_are_native_histogram_ranges(&self, schema: &DFSchemaRef) -> bool {
!self.ctx.field_columns.is_empty()
&& self
.ctx
.field_columns
.iter()
.all(|field| Self::field_column_is_native_histogram_range(schema, field))
}
fn optional_tsid_projection(
schema: &DFSchemaRef,
table_ref: Option<&TableReference>,
@@ -4719,6 +5225,9 @@ enum ScalarFunc {
/// Similar to DataFusionBuiltin but for custom functions not built into DataFusion.
/// Processing: Direct pass-through with argument positioning.
DataFusionUdf(Arc<ScalarUdfDef>),
/// Native histogram helper UDFs. Non-histogram inputs are projected as NULL
/// so the normal PromQL empty-value filter drops them.
NativeHistogramUdf(Arc<ScalarUdfDef>),
/// PromQL-specific functions that operate on time series data with temporal context.
/// These functions require both timestamp ranges and values to perform calculations.
/// Processing: Automatically injects timestamp_range and value columns as first arguments.
@@ -4745,6 +5254,7 @@ mod test {
use catalog::memory::{MemoryCatalogManager, new_memory_catalog_manager};
use common_base::Plugins;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_query::native_histogram::NATIVE_HISTOGRAM_FIELD;
use common_query::prelude::greptime_timestamp;
use common_query::test_util::DummyDecoder;
use datafusion::arrow::datatypes::Schema as ArrowSchema;
@@ -4925,6 +5435,62 @@ mod test {
)
}
async fn build_test_native_histogram_table_provider(table_name: &str) -> DfTableSourceProvider {
let catalog_list = MemoryCatalogManager::with_default_setup();
let columns = vec![
ColumnSchema::new(
"tag_0".to_string(),
ConcreteDataType::string_datatype(),
false,
),
ColumnSchema::new(
"timestamp".to_string(),
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new(
NATIVE_HISTOGRAM_FIELD.to_string(),
native_histogram_value_type().clone(),
true,
),
];
let schema = Arc::new(Schema::new(columns));
let table_meta = TableMetaBuilder::empty()
.schema(schema)
.primary_key_indices(vec![0])
.value_indices(vec![2])
.next_column_id(1024)
.build()
.unwrap();
let table_info = TableInfoBuilder::default()
.name(table_name)
.meta(table_meta)
.build()
.unwrap();
let table = EmptyTable::from_table_info(&table_info);
assert!(
catalog_list
.register_table_sync(RegisterTableRequest {
catalog: DEFAULT_CATALOG_NAME.to_string(),
schema: DEFAULT_SCHEMA_NAME.to_string(),
table_name: table_name.to_string(),
table_id: 1024,
table,
})
.is_ok()
);
DfTableSourceProvider::new(
catalog_list,
false,
QueryContext::arc(),
DummyDecoder::arc(),
false,
)
}
async fn build_test_table_provider_with_tsid(
table_name_tuples: &[(String, String)],
num_tag: usize,
@@ -6278,6 +6844,7 @@ mod test {
let mut planner = PromPlanner {
table_provider,
ctx: PromPlannerContext::from_eval_stmt(&eval_stmt),
promql_annotations: None,
};
let plan = planner
.prom_expr_to_plan(&eval_stmt.expr, &build_query_engine_state())
@@ -7132,6 +7699,78 @@ mod test {
indie_query_plan_compare(query, expected).await;
}
async fn native_histogram_plan(query: &str) -> String {
let table_provider = build_test_native_histogram_table_provider("some_metric").await;
let plan = PromPlanner::stmt_to_plan(
table_provider,
&build_eval_stmt(query),
&build_query_engine_state(),
)
.await
.unwrap();
plan.display_indent_schema().to_string()
}
#[tokio::test]
async fn native_histogram_count_uses_native_udf() {
let plan = native_histogram_plan("histogram_count(some_metric)").await;
assert!(plan.contains("prom_native_histogram_count"), "{plan}");
assert!(!plan.contains("PromHistogramFold"), "{plan}");
}
#[tokio::test]
async fn native_histogram_rate_can_feed_count() {
let plan = native_histogram_plan("histogram_count(rate(some_metric[5m]))").await;
assert!(plan.contains("prom_native_histogram_rate"), "{plan}");
assert!(plan.contains("prom_native_histogram_count"), "{plan}");
}
#[tokio::test]
async fn native_histogram_quantile_skips_classic_fold() {
let plan = native_histogram_plan("histogram_quantile(0.9, some_metric)").await;
assert!(plan.contains("prom_native_histogram_quantile"), "{plan}");
assert!(!plan.contains("PromHistogramFold"), "{plan}");
}
#[tokio::test]
async fn native_histogram_topk_uses_drop_udf() {
let plan = native_histogram_plan("topk(1, some_metric)").await;
assert!(plan.contains("prom_native_histogram_drop_float"), "{plan}");
}
#[tokio::test]
async fn unsupported_native_histogram_functions_use_drop_udf() {
for query in [
"deriv(some_metric[5m])",
"min_over_time(some_metric[5m])",
"quantile_over_time(0.9, some_metric[5m])",
"predict_linear(some_metric[5m], 60)",
"round(some_metric)",
"abs(some_metric)",
] {
let plan = native_histogram_plan(query).await;
assert!(
plan.contains("prom_native_histogram_drop_float"),
"{query}\n{plan}"
);
}
}
#[tokio::test]
async fn native_histogram_absent_over_time_uses_native_udf() {
let plan = native_histogram_plan("absent_over_time(some_metric[5m])").await;
assert!(
plan.contains("prom_native_histogram_absent_over_time"),
"{plan}"
);
}
#[tokio::test]
async fn less_filter_on_value() {
let query = "some_metric < 1.2345";
+14
View File
@@ -74,17 +74,31 @@ use crate::prometheus_handler::PrometheusHandlerRef;
/// For [ValueType::Vector] result type
#[derive(Debug, Default, Serialize, Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct PromSeriesVector {
pub metric: BTreeMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub value: Option<(f64, String)>,
#[serde(skip_serializing_if = "Option::is_none")]
pub histogram: Option<(f64, PromNativeHistogram)>,
}
/// For [ValueType::Matrix] result type
#[derive(Debug, Default, Serialize, Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct PromSeriesMatrix {
pub metric: BTreeMap<String, String>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub values: Vec<(f64, String)>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub histograms: Vec<(f64, PromNativeHistogram)>,
}
#[derive(Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct PromNativeHistogram {
pub count: String,
pub sum: String,
pub buckets: Vec<(u8, String, String, String)>,
}
/// Variants corresponding to [ValueType]
+405 -43
View File
@@ -16,7 +16,7 @@
use std::cmp::Ordering;
use std::collections::{BTreeMap, HashMap};
use arrow::array::{Array, AsArray};
use arrow::array::{Array, AsArray, StructArray};
use arrow::datatypes::{Float64Type, TimestampMillisecondType};
use arrow_schema::DataType;
use axum::Json;
@@ -24,6 +24,9 @@ use axum::http::HeaderValue;
use axum::response::{IntoResponse, Response};
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_query::native_histogram::{
NativeHistogram, is_native_histogram_value_type, read_histogram,
};
use common_query::{Output, OutputData};
use common_recordbatch::RecordBatches;
use datatypes::prelude::ConcreteDataType;
@@ -35,13 +38,37 @@ use serde_json::Value;
use snafu::{OptionExt, ResultExt};
use crate::error::{
ArrowSnafu, CollectRecordbatchSnafu, Result, UnexpectedResultSnafu, status_code_to_http_status,
ArrowSnafu, CollectRecordbatchSnafu, DataFusionSnafu, Result, UnexpectedResultSnafu,
status_code_to_http_status,
};
use crate::http::header::{GREPTIME_DB_HEADER_METRICS, collect_plan_metrics};
use crate::http::prometheus::{
PromData, PromQueryResult, PromSeriesMatrix, PromSeriesVector, PrometheusResponse,
PromData, PromNativeHistogram, PromQueryResult, PromSeriesMatrix, PromSeriesVector,
PrometheusResponse,
};
#[derive(Default)]
struct PromSeriesSamples {
values: Vec<(f64, String)>,
histograms: Vec<(f64, PromNativeHistogram)>,
}
fn prometheus_native_histogram(histogram: &NativeHistogram) -> Option<PromNativeHistogram> {
Some(PromNativeHistogram {
count: histogram.count.to_string(),
sum: histogram.sum.to_string(),
buckets: histogram.to_prometheus_buckets()?,
})
}
fn prometheus_native_histogram_or_error(
histogram: &NativeHistogram,
) -> Result<PromNativeHistogram> {
prometheus_native_histogram(histogram).context(UnexpectedResultSnafu {
reason: "native histogram cannot be converted to Prometheus buckets".to_string(),
})
}
#[derive(Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct PrometheusJsonResponse {
pub status: String,
@@ -55,6 +82,8 @@ pub struct PrometheusJsonResponse {
pub error_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub warnings: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub infos: Option<Vec<String>>,
#[serde(skip)]
pub status_code: Option<StatusCode>,
@@ -99,6 +128,7 @@ impl PrometheusJsonResponse {
error: Some(reason.into()),
error_type: Some(error_type.to_string()),
warnings: None,
infos: None,
resp_metrics: Default::default(),
status_code: Some(error_type),
}
@@ -111,6 +141,7 @@ impl PrometheusJsonResponse {
error: None,
error_type: None,
warnings: None,
infos: None,
resp_metrics: Default::default(),
status_code: None,
}
@@ -124,6 +155,7 @@ impl PrometheusJsonResponse {
) -> Self {
let response: Result<Self> = try {
let result = result?;
let mut meta = result.meta;
let mut resp =
match result.data {
OutputData::RecordBatches(batches) => Self::success(
@@ -144,8 +176,16 @@ impl PrometheusJsonResponse {
"expected data result, but got affected rows",
),
};
meta.collect_promql_annotations();
if let Some(physical_plan) = result.meta.plan {
if !meta.warnings.is_empty() {
resp.warnings = Some(meta.warnings);
}
if !meta.infos.is_empty() {
resp.infos = Some(meta.infos);
}
if let Some(physical_plan) = meta.plan {
let mut result_map = HashMap::new();
let mut tmp = vec![&mut result_map];
collect_plan_metrics(&physical_plan, &mut tmp);
@@ -199,6 +239,7 @@ impl PrometheusJsonResponse {
let mut timestamp_column_index = None;
let mut tag_column_indices = Vec::new();
let mut first_field_column_index = None;
let mut native_histogram_column_index = None;
let mut num_label_columns = 0;
@@ -224,6 +265,11 @@ impl PrometheusJsonResponse {
{
first_field_column_index = Some(i);
}
_ if native_histogram_column_index.is_none()
&& is_native_histogram_value_type(&column.data_type) =>
{
native_histogram_column_index = Some(i);
}
ConcreteDataType::String(_) => {
tag_column_indices.push(i);
num_label_columns += 1;
@@ -235,13 +281,16 @@ impl PrometheusJsonResponse {
let timestamp_column_index = timestamp_column_index.context(UnexpectedResultSnafu {
reason: "no timestamp column found".to_string(),
})?;
let first_field_column_index = first_field_column_index.context(UnexpectedResultSnafu {
reason: "no value column found".to_string(),
})?;
if first_field_column_index.is_none() && native_histogram_column_index.is_none() {
return UnexpectedResultSnafu {
reason: "no value column found".to_string(),
}
.fail();
}
// Preserves the order of output tags.
// Tag order matters, e.g., after sorc and sort_desc, the output order must be kept.
let mut buffer = IndexMap::<Vec<(&str, &str)>, Vec<(f64, String)>>::new();
let mut buffer = IndexMap::<Vec<(&str, &str)>, PromSeriesSamples>::new();
let schema = batches.schema();
for batch in batches.iter() {
@@ -258,42 +307,73 @@ impl PrometheusJsonResponse {
.column(timestamp_column_index)
.as_primitive::<TimestampMillisecondType>();
let array =
arrow::compute::cast(batch.column(first_field_column_index), &DataType::Float64)
.context(ArrowSnafu)?;
let field_column = array.as_primitive::<Float64Type>();
let field_array = first_field_column_index
.map(|index| arrow::compute::cast(batch.column(index), &DataType::Float64))
.transpose()
.context(ArrowSnafu)?;
let field_column = field_array
.as_ref()
.map(|array| array.as_primitive::<Float64Type>());
let native_histogram_column = native_histogram_column_index
.map(|index| {
batch
.column(index)
.as_any()
.downcast_ref::<StructArray>()
.with_context(|| UnexpectedResultSnafu {
reason: "native histogram column is not a struct array".to_string(),
})
})
.transpose()?;
// assemble rows
for row_index in 0..batch.num_rows() {
// retrieve value
if field_column.is_valid(row_index) {
let v = field_column.value(row_index);
// ignore all NaN values to reduce the amount of data to be sent.
if v.is_nan() {
continue;
let value = field_column.and_then(|field_column| {
if !field_column.is_valid(row_index) {
return None;
}
let value = field_column.value(row_index);
(!value.is_nan()).then_some((timestamp_column.value(row_index), value))
});
let histogram = native_histogram_column
.and_then(|column| {
read_histogram(column, row_index)
.context(DataFusionSnafu)
.transpose()
})
.transpose()?
.map(|histogram| {
prometheus_native_histogram_or_error(&histogram)
.map(|histogram| (timestamp_column.value(row_index), histogram))
})
.transpose()?;
// retrieve tags
let mut tags = Vec::with_capacity(num_label_columns + 1);
if let Some(metric_name) = &metric_name {
tags.push((METRIC_NAME, metric_name.as_str()));
}
for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) {
// TODO(ruihang): add test for NULL tag
if tag_column.is_valid(row_index) {
tags.push((tag_name, tag_column.value(row_index)));
}
if value.is_none() && histogram.is_none() {
continue;
}
// retrieve tags
let mut tags = Vec::with_capacity(num_label_columns + 1);
if let Some(metric_name) = &metric_name {
tags.push((METRIC_NAME, metric_name.as_str()));
}
for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) {
// TODO(ruihang): add test for NULL tag
if tag_column.is_valid(row_index) {
tags.push((tag_name, tag_column.value(row_index)));
}
}
// retrieve timestamp
let timestamp_millis = timestamp_column.value(row_index);
let timestamp = timestamp_millis as f64 / 1000.0;
buffer
.entry(tags)
.or_default()
.push((timestamp, Into::<f64>::into(v).to_string()));
};
let entry = buffer.entry(tags).or_default();
if let Some((timestamp_millis, histogram)) = histogram {
entry
.histograms
.push((timestamp_millis as f64 / 1000.0, histogram));
} else if let Some((timestamp_millis, value)) = value {
entry
.values
.push((timestamp_millis as f64 / 1000.0, value.to_string()));
}
}
}
@@ -306,28 +386,46 @@ impl PrometheusJsonResponse {
};
// accumulate data into result
buffer.into_iter().for_each(|(tags, mut values)| {
buffer.into_iter().for_each(|(tags, mut samples)| {
let metric = tags
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect::<BTreeMap<_, _>>();
match result {
PromQueryResult::Vector(ref mut v) => {
let histogram = samples.histograms.pop();
let value = if histogram.is_none() {
samples.values.pop()
} else {
None
};
v.push(PromSeriesVector {
metric,
value: values.pop(),
value,
histogram,
});
}
PromQueryResult::Matrix(ref mut v) => {
// sort values by timestamp
if !values.is_sorted_by(|a, b| a.0 <= b.0) {
values.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Equal));
if !samples.values.is_sorted_by(|a, b| a.0 <= b.0) {
samples
.values
.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Equal));
}
if !samples.histograms.is_sorted_by(|a, b| a.0 <= b.0) {
samples
.histograms
.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Equal));
}
v.push(PromSeriesMatrix { metric, values });
v.push(PromSeriesMatrix {
metric,
values: samples.values,
histograms: samples.histograms,
});
}
PromQueryResult::Scalar(ref mut v) => {
*v = values.pop();
*v = samples.values.pop();
}
PromQueryResult::String(ref mut _v) => {
// TODO(ruihang): Not supported yet
@@ -350,3 +448,267 @@ impl PrometheusJsonResponse {
Ok(data)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use common_query::native_histogram::{
NATIVE_HISTOGRAM_FIELD, NativeHistogram, Span, build_histogram_array,
native_histogram_value_type,
};
use common_query::{Output, OutputData, OutputMeta};
use common_recordbatch::{RecordBatch, RecordBatches};
use datatypes::data_type::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{
Float64Vector, StringVector, StructVector, TimestampMillisecondVector, VectorRef,
};
use super::*;
fn sample_histogram() -> NativeHistogram {
NativeHistogram {
schema: 0,
zero_threshold: 0.001,
sum: 3.0,
reset_hint: 0,
start_timestamp: Some(0),
custom_values: vec![],
positive_spans: vec![Span {
offset: 0,
length: 1,
}],
negative_spans: vec![],
count: 2.0,
zero_count: 1.0,
positive_buckets: vec![1.0],
negative_buckets: vec![],
}
}
fn histogram_vector(values: &[Option<NativeHistogram>]) -> VectorRef {
let histogram_array = build_histogram_array(values);
let histogram_array = histogram_array
.as_any()
.downcast_ref::<StructArray>()
.unwrap()
.clone();
let ConcreteDataType::Struct(histogram_type) = native_histogram_value_type().clone() else {
unreachable!("native histogram type must be a struct")
};
Arc::new(StructVector::try_new(histogram_type, histogram_array).unwrap())
}
#[tokio::test]
async fn from_query_result_preserves_warnings_and_infos() {
let recordbatches = RecordBatches::empty();
let mut meta = OutputMeta::default();
meta.warnings.push("warn".to_string());
meta.infos.push("info".to_string());
let response = PrometheusJsonResponse::from_query_result(
Ok(Output::new(OutputData::RecordBatches(recordbatches), meta)),
None,
ValueType::Vector,
)
.await;
assert_eq!(response.warnings, Some(vec!["warn".to_string()]));
assert_eq!(response.infos, Some(vec!["info".to_string()]));
}
#[test]
fn record_batches_to_data_serializes_aliased_native_histogram_vector() {
let schema = Arc::new(Schema::new(vec![
ColumnSchema::new(
"greptime_timestamp",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new("job", ConcreteDataType::string_datatype(), false),
ColumnSchema::new(
"prom_native_histogram_rate(greptime_native_histogram)",
native_histogram_value_type().clone(),
true,
),
]));
let columns: Vec<VectorRef> = vec![
Arc::new(TimestampMillisecondVector::from_values([1000])),
Arc::new(StringVector::from(vec![Some("api")])),
histogram_vector(&[Some(sample_histogram())]),
];
let recordbatch = RecordBatch::new(schema.clone(), columns).unwrap();
let recordbatches = RecordBatches::try_new(schema, vec![recordbatch]).unwrap();
let response = PrometheusJsonResponse::record_batches_to_data(
recordbatches,
Some("http_requests".to_string()),
ValueType::Vector,
)
.unwrap();
let PrometheusResponse::PromData(PromData {
result: PromQueryResult::Vector(series),
..
}) = response
else {
panic!("expected vector response");
};
assert_eq!(series.len(), 1);
assert_eq!(series[0].metric["__name__"], "http_requests");
assert_eq!(series[0].metric["job"], "api");
assert!(series[0].value.is_none());
let (timestamp, histogram) = series[0].histogram.as_ref().unwrap();
assert_eq!(*timestamp, 1.0);
assert_eq!(histogram.count, "2");
assert_eq!(histogram.sum, "3");
assert_eq!(
histogram.buckets,
vec![
(
3,
"-0.001".to_string(),
"0.001".to_string(),
"1".to_string()
),
(0, "0.5".to_string(), "1".to_string(), "1".to_string()),
]
);
}
#[test]
fn record_batches_to_data_prefers_histogram_for_instant_vector() {
let schema = Arc::new(Schema::new(vec![
ColumnSchema::new(
"greptime_timestamp",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new("job", ConcreteDataType::string_datatype(), false),
ColumnSchema::new("value", ConcreteDataType::float64_datatype(), true),
ColumnSchema::new(
NATIVE_HISTOGRAM_FIELD,
native_histogram_value_type().clone(),
true,
),
]));
let columns: Vec<VectorRef> = vec![
Arc::new(TimestampMillisecondVector::from_values([1000])),
Arc::new(StringVector::from(vec![Some("api")])),
Arc::new(Float64Vector::from_slice([42.0])),
histogram_vector(&[Some(sample_histogram())]),
];
let recordbatch = RecordBatch::new(schema.clone(), columns).unwrap();
let recordbatches = RecordBatches::try_new(schema, vec![recordbatch]).unwrap();
let response = PrometheusJsonResponse::record_batches_to_data(
recordbatches,
Some("http_requests".to_string()),
ValueType::Vector,
)
.unwrap();
let PrometheusResponse::PromData(PromData {
result: PromQueryResult::Vector(series),
..
}) = response
else {
panic!("expected vector response");
};
assert_eq!(series.len(), 1);
assert!(series[0].value.is_none());
assert!(series[0].histogram.is_some());
}
#[test]
fn record_batches_to_data_prefers_histogram_for_matrix_row() {
let schema = Arc::new(Schema::new(vec![
ColumnSchema::new(
"greptime_timestamp",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new("job", ConcreteDataType::string_datatype(), false),
ColumnSchema::new("value", ConcreteDataType::float64_datatype(), true),
ColumnSchema::new(
NATIVE_HISTOGRAM_FIELD,
native_histogram_value_type().clone(),
true,
),
]));
let columns: Vec<VectorRef> = vec![
Arc::new(TimestampMillisecondVector::from_values([1000, 2000])),
Arc::new(StringVector::from(vec![Some("api"), Some("api")])),
Arc::new(Float64Vector::from_slice([42.0, 7.0])),
histogram_vector(&[Some(sample_histogram()), None]),
];
let recordbatch = RecordBatch::new(schema.clone(), columns).unwrap();
let recordbatches = RecordBatches::try_new(schema, vec![recordbatch]).unwrap();
let response = PrometheusJsonResponse::record_batches_to_data(
recordbatches,
Some("http_requests".to_string()),
ValueType::Matrix,
)
.unwrap();
let PrometheusResponse::PromData(PromData {
result: PromQueryResult::Matrix(series),
..
}) = response
else {
panic!("expected matrix response");
};
assert_eq!(series.len(), 1);
assert_eq!(series[0].values, vec![(2.0, "7".to_string())]);
assert_eq!(series[0].histograms.len(), 1);
assert_eq!(series[0].histograms[0].0, 1.0);
}
#[test]
fn record_batches_to_data_errors_on_unrenderable_native_histogram() {
let schema = Arc::new(Schema::new(vec![
ColumnSchema::new(
"greptime_timestamp",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new("job", ConcreteDataType::string_datatype(), false),
ColumnSchema::new(
NATIVE_HISTOGRAM_FIELD,
native_histogram_value_type().clone(),
true,
),
]));
let mut histogram = sample_histogram();
histogram.schema = 100;
let columns: Vec<VectorRef> = vec![
Arc::new(TimestampMillisecondVector::from_values([1000])),
Arc::new(StringVector::from(vec![Some("api")])),
histogram_vector(&[Some(histogram)]),
];
let recordbatch = RecordBatch::new(schema.clone(), columns).unwrap();
let recordbatches = RecordBatches::try_new(schema, vec![recordbatch]).unwrap();
let error = PrometheusJsonResponse::record_batches_to_data(
recordbatches,
Some("http_requests".to_string()),
ValueType::Vector,
)
.unwrap_err();
assert!(
error
.to_string()
.contains("native histogram cannot be converted to Prometheus buckets"),
"{error}"
);
}
}
+380 -22
View File
@@ -16,9 +16,11 @@ use std::collections::hash_map::Entry;
use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
use api::greptime_proto::io::prometheus::write::v2::histogram::{Count, ZeroCount};
use api::greptime_proto::io::prometheus::write::v2::{
BucketSpan, Histogram, Request, Sample, TimeSeries,
};
#[cfg(test)]
use api::greptime_proto::io::prometheus::write::v2::{Exemplar, Metadata, metadata};
use api::greptime_proto::io::prometheus::write::v2::{Histogram, Request, Sample, TimeSeries};
use api::helper::ColumnDataTypeWrapper;
use api::v1::value::ValueData;
use api::v1::{ColumnSchema, ListValue, RowInsertRequest, Rows, SemanticType, Value};
@@ -43,6 +45,9 @@ use crate::row_writer::{self, TableData};
type PromTags = Vec<(String, String)>;
type ResolvedSeriesLabels = (PromCtx, String, PromTags);
const MIN_REMOTE_WRITE_V2_SCHEMA: i32 = -4;
const MAX_REMOTE_WRITE_V2_SCHEMA: i32 = 8;
const MAX_REDUCIBLE_REMOTE_WRITE_V2_SCHEMA: i32 = 52;
pub(crate) fn decode_remote_write_v2_request(is_zstd: bool, body: Bytes) -> Result<Request> {
let _timer = crate::metrics::METRIC_HTTP_PROM_STORE_DECODE_ELAPSED.start_timer();
@@ -279,6 +284,9 @@ fn native_histogram_column_schema() -> ColumnSchema {
}
fn native_histogram_struct_value(histogram: &Histogram) -> Result<ValueData> {
let uses_float_counts = native_histogram_uses_float_counts(histogram)?;
validate_native_histogram(histogram, uses_float_counts)?;
let mut items = Vec::with_capacity(NATIVE_HISTOGRAM_FIELD_NAMES.len());
items.extend([
pb_value(ValueData::I32Value(histogram.schema)),
@@ -295,39 +303,39 @@ fn native_histogram_struct_value(histogram: &Histogram) -> Result<ValueData> {
u32_list_value(histogram.negative_spans.iter().map(|span| span.length)),
]);
let int_counts = match (&histogram.count, &histogram.zero_count) {
(Some(Count::CountInt(count)), Some(ZeroCount::ZeroCountInt(zero_count))) => {
(*count, *zero_count)
}
(Some(Count::CountInt(count)), _) => (*count, 0),
(_, Some(ZeroCount::ZeroCountInt(zero_count))) => (0, *zero_count),
_ => (0, 0),
};
let float_counts = match (&histogram.count, &histogram.zero_count) {
(Some(Count::CountFloat(count)), Some(ZeroCount::ZeroCountFloat(zero_count))) => {
Some((*count, *zero_count))
}
(Some(Count::CountFloat(count)), _) => Some((*count, 0.0)),
_ => None,
};
if let Some(counts) = float_counts {
if uses_float_counts {
let count = match histogram.count.as_ref() {
Some(Count::CountFloat(count)) => *count,
_ => 0.0,
};
let zero_count = match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count,
_ => 0.0,
};
items.extend([
null_pb_value(),
null_pb_value(),
i64_list_value(std::iter::empty()),
i64_list_value(std::iter::empty()),
pb_value(ValueData::F64Value(counts.0)),
pb_value(ValueData::F64Value(counts.1)),
pb_value(ValueData::F64Value(count)),
pb_value(ValueData::F64Value(zero_count)),
f64_list_value(histogram.positive_counts.iter().copied()),
f64_list_value(histogram.negative_counts.iter().copied()),
]);
} else {
let count = match histogram.count.as_ref() {
Some(Count::CountInt(count)) => *count,
_ => 0,
};
let zero_count = match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count,
_ => 0,
};
let positive_buckets = bucket_counts_from_deltas(&histogram.positive_deltas)?;
let negative_buckets = bucket_counts_from_deltas(&histogram.negative_deltas)?;
items.extend([
pb_value(ValueData::U64Value(int_counts.0)),
pb_value(ValueData::U64Value(int_counts.1)),
pb_value(ValueData::U64Value(count)),
pb_value(ValueData::U64Value(zero_count)),
i64_list_value(positive_buckets.iter().copied()),
i64_list_value(negative_buckets.iter().copied()),
null_pb_value(),
@@ -340,6 +348,229 @@ fn native_histogram_struct_value(histogram: &Histogram) -> Result<ValueData> {
Ok(ValueData::StructValue(api::v1::StructValue { items }))
}
fn validate_native_histogram(histogram: &Histogram, uses_float_counts: bool) -> Result<()> {
validate_native_histogram_schema(histogram.schema)?;
validate_native_histogram_custom_values(histogram)?;
let (positive_buckets, negative_buckets) = if uses_float_counts {
(
histogram.positive_counts.len(),
histogram.negative_counts.len(),
)
} else {
(
histogram.positive_deltas.len(),
histogram.negative_deltas.len(),
)
};
let custom_max_index =
(histogram.schema == CUSTOM_BUCKETS_SCHEMA).then_some(histogram.custom_values.len() as i32);
validate_native_histogram_spans(
"positive",
&histogram.positive_spans,
positive_buckets,
custom_max_index,
)?;
validate_native_histogram_spans(
"negative",
&histogram.negative_spans,
negative_buckets,
None,
)?;
if histogram.schema == CUSTOM_BUCKETS_SCHEMA {
ensure!(
histogram.zero_threshold == 0.0 && native_histogram_zero_count_is_zero(histogram),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram must not use a zero bucket"
.to_string(),
}
);
ensure!(
histogram.negative_spans.is_empty()
&& histogram.negative_deltas.is_empty()
&& histogram.negative_counts.is_empty(),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram must not use negative buckets"
.to_string(),
}
);
}
Ok(())
}
fn validate_native_histogram_schema(schema: i32) -> Result<()> {
if schema == CUSTOM_BUCKETS_SCHEMA
|| (MIN_REMOTE_WRITE_V2_SCHEMA..=MAX_REMOTE_WRITE_V2_SCHEMA).contains(&schema)
{
return Ok(());
}
if (MAX_REMOTE_WRITE_V2_SCHEMA + 1..=MAX_REDUCIBLE_REMOTE_WRITE_V2_SCHEMA).contains(&schema) {
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram schema {schema} must be reduced before ingestion"
),
}
.fail()
} else {
error::InvalidPromRemoteRequestSnafu {
msg: format!("remote write v2 native histogram schema {schema} is unsupported"),
}
.fail()
}
}
fn validate_native_histogram_custom_values(histogram: &Histogram) -> Result<()> {
if histogram.schema != CUSTOM_BUCKETS_SCHEMA {
ensure!(
histogram.custom_values.is_empty(),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 standard native histogram must not use custom_values"
.to_string(),
}
);
return Ok(());
}
for value in &histogram.custom_values {
ensure!(
!value.is_nan() && *value != f64::INFINITY,
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram custom_values must not contain +Inf or NaN"
.to_string(),
}
);
}
for values in histogram.custom_values.windows(2) {
ensure!(
values[0] < values[1],
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram custom_values must be sorted"
.to_string(),
}
);
}
Ok(())
}
fn validate_native_histogram_spans(
name: &str,
spans: &[BucketSpan],
bucket_count: usize,
custom_max_index: Option<i32>,
) -> Result<()> {
let span_len = spans
.iter()
.try_fold(0usize, |sum, span| sum.checked_add(span.length as usize))
.context(error::InvalidPromRemoteRequestSnafu {
msg: format!("remote write v2 native histogram {name} spans overflow"),
})?;
ensure!(
span_len == bucket_count,
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} spans describe {span_len} buckets, found {bucket_count}"
),
}
);
let mut current_index = 0i32;
let mut first = true;
for span in spans {
current_index = if first {
first = false;
span.offset
} else {
current_index.checked_add(span.offset).context(
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} span index overflows i32"
),
},
)?
};
for _ in 0..span.length {
if let Some(max_index) = custom_max_index {
ensure!(
(0..=max_index).contains(&current_index),
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 custom native histogram {name} bucket index {current_index} is out of range"
),
}
);
}
current_index =
current_index
.checked_add(1)
.context(error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} span index overflows i32"
),
})?;
}
}
Ok(())
}
fn native_histogram_zero_count_is_zero(histogram: &Histogram) -> bool {
match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count == 0,
Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count == 0.0,
None => true,
}
}
fn native_histogram_uses_float_counts(histogram: &Histogram) -> Result<bool> {
let uses_float_count = matches!(histogram.count, Some(Count::CountFloat(_)))
|| matches!(histogram.zero_count, Some(ZeroCount::ZeroCountFloat(_)));
let uses_int_count = matches!(histogram.count, Some(Count::CountInt(_)))
|| matches!(histogram.zero_count, Some(ZeroCount::ZeroCountInt(_)));
let uses_float_buckets =
!histogram.positive_counts.is_empty() || !histogram.negative_counts.is_empty();
let uses_int_buckets =
!histogram.positive_deltas.is_empty() || !histogram.negative_deltas.is_empty();
if matches!(
(&histogram.count, &histogram.zero_count),
(Some(Count::CountInt(_)), Some(ZeroCount::ZeroCountFloat(_)))
| (Some(Count::CountFloat(_)), Some(ZeroCount::ZeroCountInt(_)))
) {
return error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram count and zero_count must use the same integer or float family".to_string(),
}
.fail();
}
ensure!(
!(uses_float_buckets && uses_int_buckets),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram bucket counts must use either integer deltas or float counts".to_string(),
}
);
ensure!(
!(uses_float_count && uses_int_buckets),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 float native histogram must not use integer bucket deltas"
.to_string(),
}
);
ensure!(
!(uses_int_count && uses_float_buckets),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 integer native histogram must not use float bucket counts"
.to_string(),
}
);
Ok(uses_float_count || uses_float_buckets)
}
fn pb_value(value_data: ValueData) -> Value {
optional_pb_value(Some(value_data))
}
@@ -859,6 +1090,126 @@ mod tests {
"conflicts with an internal native histogram label",
));
cases.push((
"int count with float zero count",
request_with_histogram(Histogram {
count: Some(Count::CountInt(1)),
zero_count: Some(ZeroCount::ZeroCountFloat(0.5)),
..Default::default()
}),
"count and zero_count must use the same integer or float family",
));
cases.push((
"float count with int zero count",
request_with_histogram(Histogram {
count: Some(Count::CountFloat(1.0)),
zero_count: Some(ZeroCount::ZeroCountInt(1)),
..Default::default()
}),
"count and zero_count must use the same integer or float family",
));
cases.push((
"reducible schema",
request_with_histogram(Histogram {
schema: 9,
..Default::default()
}),
"schema 9 must be reduced before ingestion",
));
cases.push((
"unsupported schema",
request_with_histogram(Histogram {
schema: 53,
..Default::default()
}),
"schema 53 is unsupported",
));
cases.push((
"standard schema with custom values",
request_with_histogram(Histogram {
schema: 1,
custom_values: vec![1.0],
..Default::default()
}),
"standard native histogram must not use custom_values",
));
cases.push((
"custom values with inf",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![f64::INFINITY],
..Default::default()
}),
"custom_values must not contain +Inf or NaN",
));
cases.push((
"custom values not sorted",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![2.0, 1.0],
..Default::default()
}),
"custom_values must be sorted",
));
cases.push((
"custom schema with zero bucket",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
zero_count: Some(ZeroCount::ZeroCountInt(1)),
..Default::default()
}),
"custom native histogram must not use a zero bucket",
));
cases.push((
"custom schema with negative buckets",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
negative_spans: vec![BucketSpan {
offset: -1,
length: 1,
}],
negative_deltas: vec![1],
..Default::default()
}),
"custom native histogram must not use negative buckets",
));
cases.push((
"span count mismatch",
request_with_histogram(Histogram {
positive_spans: vec![BucketSpan {
offset: 0,
length: 2,
}],
positive_deltas: vec![1],
..Default::default()
}),
"positive spans describe 2 buckets, found 1",
));
cases.push((
"custom span index out of range",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![1.0],
positive_spans: vec![BucketSpan {
offset: 2,
length: 1,
}],
positive_deltas: vec![1],
..Default::default()
}),
"positive bucket index 2 is out of range",
));
for (name, request, expected) in cases {
assert_invalid(name, request, expected);
}
@@ -1115,6 +1466,13 @@ mod tests {
)
}
fn request_with_histogram(histogram: Histogram) -> Request {
test_util::request_with_labels_and_histograms(
vec![(METRIC_NAME_LABEL, "metric")],
vec![histogram],
)
}
fn assert_invalid(name: &str, request: Request, expected: &str) {
let err = match into_write_requests(request) {
Ok(_) => panic!("{name}: expected invalid request error"),
+5
View File
@@ -872,6 +872,7 @@ pub async fn test_prom_gateway_query(store_type: StorageType) {
.into_iter()
.collect(),
value: Some((5.0, "1".to_string())),
..Default::default()
},
PromSeriesVector {
metric: [
@@ -881,6 +882,7 @@ pub async fn test_prom_gateway_query(store_type: StorageType) {
.into_iter()
.collect(),
value: Some((5.0, "2".to_string())),
..Default::default()
},
]
);
@@ -932,6 +934,7 @@ pub async fn test_prom_gateway_query(store_type: StorageType) {
.into_iter()
.collect(),
values: vec![(5.0, "1".to_string()), (10.0, "1".to_string())],
..Default::default()
},
PromSeriesMatrix {
metric: [
@@ -941,6 +944,7 @@ pub async fn test_prom_gateway_query(store_type: StorageType) {
.into_iter()
.collect(),
values: vec![(5.0, "2".to_string()), (10.0, "2".to_string())],
..Default::default()
},
]
);
@@ -973,6 +977,7 @@ pub async fn test_prom_gateway_query(store_type: StorageType) {
error: None,
error_type: None,
warnings: None,
infos: None,
resp_metrics: Default::default(),
status_code: None,
};
+49 -2
View File
@@ -2455,7 +2455,6 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag
reset_hint: 2,
timestamp: 3000,
start_timestamp: 1500,
custom_values: vec![0.5, 1.5],
..Default::default()
},
Histogram {
@@ -2508,10 +2507,58 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag
"prometheus_remote_write_v2_native_histogram_rows",
&client,
"select greptime_timestamp, greptime_native_histogram, job, instance from remote_write_v2_latency_seconds order by greptime_timestamp;",
"[[3000,{\"count_f64\":null,\"count_u64\":4,\"custom_values\":[0.5,1.5],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_u64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":3.5,\"count_u64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_u64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]",
"[[3000,{\"count_f64\":null,\"count_u64\":4,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_u64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":3.5,\"count_u64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_u64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]",
)
.await;
let res = client
.get("/v1/prometheus/api/v1/query?query=histogram_count(remote_write_v2_latency_seconds)&time=4")
.send()
.await;
assert_eq!(res.status(), StatusCode::OK);
let body = serde_json::from_str::<Value>(&res.text().await).unwrap();
assert_eq!(body["status"], "success");
assert_eq!(body["data"]["resultType"], "vector");
let series = body["data"]["result"].as_array().unwrap();
assert_eq!(series.len(), 1);
let value = series[0]["value"].as_array().unwrap();
assert_eq!(value[0].as_f64(), Some(4.0));
assert_eq!(value[1], "3.5");
let res = client
.get("/v1/prometheus/api/v1/query?query=remote_write_v2_latency_seconds&time=4")
.send()
.await;
assert_eq!(res.status(), StatusCode::OK);
let body = serde_json::from_str::<Value>(&res.text().await).unwrap();
assert_eq!(body["status"], "success");
assert_eq!(body["data"]["resultType"], "vector");
let series = body["data"]["result"].as_array().unwrap();
assert_eq!(series.len(), 1);
assert!(series[0].get("value").is_none());
let histogram = series[0]["histogram"].as_array().unwrap();
assert_eq!(histogram[0].as_f64(), Some(4.0));
assert_eq!(histogram[1]["count"], "3.5");
assert_eq!(histogram[1]["sum"], "20");
let res = client
.get("/v1/prometheus/api/v1/query_range?query=remote_write_v2_latency_seconds&start=3&end=4&step=1")
.send()
.await;
assert_eq!(res.status(), StatusCode::OK);
let body = serde_json::from_str::<Value>(&res.text().await).unwrap();
assert_eq!(body["status"], "success");
assert_eq!(body["data"]["resultType"], "matrix");
let series = body["data"]["result"].as_array().unwrap();
assert_eq!(series.len(), 1);
assert!(series[0].get("values").is_none());
let histograms = series[0]["histograms"].as_array().unwrap();
assert_eq!(histograms.len(), 2);
assert_eq!(histograms[0][0].as_f64(), Some(3.0));
assert_eq!(histograms[0][1]["count"], "4");
assert_eq!(histograms[1][0].as_f64(), Some(4.0));
assert_eq!(histograms[1][1]["count"], "3.5");
validate_data(
"prometheus_remote_write_v2_native_histogram_table_created",
&client,