1use std::any::Any;
18use std::hash::{Hash, Hasher};
19use std::mem::size_of;
20use std::sync::Arc;
21
22use common_query::native_histogram::*;
23use common_query::prometheus::format_prometheus_float;
24use common_query::promql_annotations::PromqlAnnotationCollector;
25use datafusion::arrow::array::{
26 Array, ArrayRef, BooleanArray, Float64Array, Float64Builder, Int64Array, StringBuilder,
27 StructArray, TimestampMillisecondArray, UInt64Array,
28};
29use datafusion::arrow::compute::filter;
30use datafusion::arrow::datatypes::{DataType, Field, TimeUnit};
31use datafusion::common::{DataFusionError, Result as DfResult};
32use datafusion::logical_expr::{Accumulator as DfAccumulator, AggregateUDF, ScalarUDF, Volatility};
33use datafusion::physical_plan::ColumnarValue;
34use datafusion_common::ScalarValue;
35use datafusion_expr::function::AccumulatorArgs;
36use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, create_udaf, create_udf};
37
38use crate::functions::{
39 AvgOverTime, Deriv, DoubleExponentialSmoothing, IDelta, Increase, LastOverTime, MaxOverTime,
40 MinOverTime, PredictLinear, QuantileOverTime, Rate, StddevOverTime, StdvarOverTime,
41 SumOverTime, extract_array, extract_range_dict,
42};
43use crate::range_array::{RangeArray, unpack};
44
45fn extract_histogram_array(value: &ColumnarValue, func_name: &str) -> DfResult<ArrayRef> {
46 let array = extract_array(value)?;
47 if array.data_type() != &native_histogram_arrow_type() {
48 return Err(DataFusionError::Execution(format!(
49 "{func_name}: expected native histogram struct, found {}",
50 array.data_type()
51 )));
52 }
53 Ok(array)
54}
55
56fn read_scalar_f64_arg(
57 value: &ColumnarValue,
58 row: usize,
59 len: usize,
60 func_name: &str,
61) -> DfResult<f64> {
62 match value {
63 ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok(value.unwrap_or(f64::NAN)),
64 ColumnarValue::Array(array) => {
65 let array = array
66 .as_any()
67 .downcast_ref::<Float64Array>()
68 .ok_or_else(|| {
69 DataFusionError::Execution(format!(
70 "{func_name}: expected Float64 argument, found {}",
71 array.data_type()
72 ))
73 })?;
74 if array.len() != len {
75 return Err(DataFusionError::Execution(format!(
76 "{func_name}: Float64 argument length mismatch: {} vs {len}",
77 array.len()
78 )));
79 }
80 Ok(if array.is_null(row) {
81 f64::NAN
82 } else {
83 array.value(row)
84 })
85 }
86 other => Err(DataFusionError::Execution(format!(
87 "{func_name}: expected Float64 argument, found {}",
88 other.data_type()
89 ))),
90 }
91}
92
93#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
94enum AnnotationReturn {
95 FloatNull,
96 BooleanTrue,
97 BooleanFalse,
98}
99
100#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
101enum AnnotationLevel {
102 Info,
103 Warning,
104}
105
106impl AnnotationReturn {
107 fn data_type(self) -> DataType {
108 match self {
109 Self::FloatNull => DataType::Float64,
110 Self::BooleanTrue | Self::BooleanFalse => DataType::Boolean,
111 }
112 }
113
114 fn scalar_value(self) -> ScalarValue {
115 match self {
116 Self::FloatNull => ScalarValue::Float64(None),
117 Self::BooleanTrue => ScalarValue::Boolean(Some(true)),
118 Self::BooleanFalse => ScalarValue::Boolean(Some(false)),
119 }
120 }
121}
122
123#[derive(Debug, Clone)]
124struct NativeHistogramAnnotationUdf {
125 name: &'static str,
126 signature: Signature,
127 return_kind: AnnotationReturn,
128 level: AnnotationLevel,
129 message: String,
130 collector: Option<PromqlAnnotationCollector>,
131}
132
133impl NativeHistogramAnnotationUdf {
134 fn new(
135 name: &'static str,
136 return_kind: AnnotationReturn,
137 level: AnnotationLevel,
138 message: String,
139 collector: Option<PromqlAnnotationCollector>,
140 ) -> Self {
141 Self {
142 name,
143 signature: Signature::variadic_any(Volatility::Volatile),
144 return_kind,
145 level,
146 message,
147 collector,
148 }
149 }
150}
151
152impl PartialEq for NativeHistogramAnnotationUdf {
153 fn eq(&self, other: &Self) -> bool {
154 self.name == other.name
155 && self.return_kind == other.return_kind
156 && self.level == other.level
157 && self.message == other.message
158 }
159}
160
161impl Eq for NativeHistogramAnnotationUdf {}
162
163impl Hash for NativeHistogramAnnotationUdf {
164 fn hash<H: Hasher>(&self, state: &mut H) {
165 self.name.hash(state);
166 self.return_kind.hash(state);
167 self.level.hash(state);
168 self.message.hash(state);
169 }
170}
171
172impl ScalarUDFImpl for NativeHistogramAnnotationUdf {
173 fn as_any(&self) -> &dyn Any {
174 self
175 }
176
177 fn name(&self) -> &str {
178 self.name
179 }
180
181 fn signature(&self) -> &Signature {
182 &self.signature
183 }
184
185 fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
186 Ok(self.return_kind.data_type())
187 }
188
189 fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult<ColumnarValue> {
190 let has_dropped_sample = !args.args.is_empty()
191 && (0..args.number_rows).any(|row| {
192 args.args.iter().all(|arg| match arg {
193 ColumnarValue::Array(array) => array.is_valid(row),
194 ColumnarValue::Scalar(value) => !value.is_null(),
195 })
196 });
197 if has_dropped_sample
198 && let Some(collector) = args
199 .config_options
200 .extensions
201 .get::<PromqlAnnotationCollector>()
202 .cloned()
203 .or_else(|| self.collector.clone())
204 {
205 match self.level {
206 AnnotationLevel::Info => collector.record_info(self.message.clone()),
207 AnnotationLevel::Warning => collector.record_warning(self.message.clone()),
208 }
209 }
210 Ok(ColumnarValue::Scalar(self.return_kind.scalar_value()))
211 }
212}
213
214pub struct NativeHistogramDrop;
215
216impl NativeHistogramDrop {
217 const fn float_null_name() -> &'static str {
218 "prom_native_histogram_drop_float"
219 }
220
221 const fn bool_false_name() -> &'static str {
222 "prom_native_histogram_drop_bool"
223 }
224
225 const fn bool_true_name() -> &'static str {
226 "prom_native_histogram_keep_bool"
227 }
228
229 pub fn float_null_udf(
230 message: String,
231 collector: Option<PromqlAnnotationCollector>,
232 ) -> ScalarUDF {
233 ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
234 Self::float_null_name(),
235 AnnotationReturn::FloatNull,
236 AnnotationLevel::Info,
237 message,
238 collector,
239 ))
240 }
241
242 pub fn bool_false_udf(
243 message: String,
244 collector: Option<PromqlAnnotationCollector>,
245 ) -> ScalarUDF {
246 ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
247 Self::bool_false_name(),
248 AnnotationReturn::BooleanFalse,
249 AnnotationLevel::Info,
250 message,
251 collector,
252 ))
253 }
254
255 pub fn bool_true_udf(
256 message: String,
257 collector: Option<PromqlAnnotationCollector>,
258 ) -> ScalarUDF {
259 ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
260 Self::bool_true_name(),
261 AnnotationReturn::BooleanTrue,
262 AnnotationLevel::Info,
263 message,
264 collector,
265 ))
266 }
267
268 pub fn warning_bool_false_udf(
269 message: String,
270 collector: Option<PromqlAnnotationCollector>,
271 ) -> ScalarUDF {
272 ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
273 Self::bool_false_name(),
274 AnnotationReturn::BooleanFalse,
275 AnnotationLevel::Warning,
276 message,
277 collector,
278 ))
279 }
280}
281
282fn record_info(collector: &Option<PromqlAnnotationCollector>, message: impl Into<String>) {
283 if let Some(collector) = collector {
284 collector.record_info(message);
285 }
286}
287
288fn record_warning(collector: &Option<PromqlAnnotationCollector>, message: impl Into<String>) {
289 if let Some(collector) = collector {
290 collector.record_warning(message);
291 }
292}
293
294fn record_custom_reconciliation(
295 collector: &Option<PromqlAnnotationCollector>,
296 name: &'static str,
297 lhs: &NativeHistogram,
298 rhs: &NativeHistogram,
299) {
300 if lhs.needs_custom_reconciliation(rhs) {
301 record_info(
302 collector,
303 format!("{name}: reconciled native histograms with different custom buckets"),
304 );
305 }
306}
307
308fn record_counter_reset_contradiction(
309 collector: &Option<PromqlAnnotationCollector>,
310 name: &'static str,
311 lhs: &NativeHistogram,
312 rhs: &NativeHistogram,
313) {
314 if lhs.counter_reset_hints_contradict(rhs) {
315 record_counter_reset_contradiction_warning(collector, name);
316 }
317}
318
319fn record_counter_reset_contradiction_warning(
320 collector: &Option<PromqlAnnotationCollector>,
321 name: &'static str,
322) {
323 record_warning(
324 collector,
325 format!("{name}: native histogram counter reset hints contradict"),
326 );
327}
328
329fn scalar_histogram_udf(
330 name: &'static str,
331 extra_input_types: Vec<DataType>,
332 calc: fn(&NativeHistogram, &[ColumnarValue], usize, usize, &'static str) -> DfResult<f64>,
333) -> ScalarUDF {
334 let mut input_types = vec![native_histogram_arrow_type()];
335 input_types.extend(extra_input_types);
336 create_udf(
337 name,
338 input_types,
339 DataType::Float64,
340 Volatility::Volatile,
341 Arc::new(move |input: &[ColumnarValue]| {
342 if input.is_empty() {
343 return Err(DataFusionError::Plan(format!(
344 "{name} requires a native histogram argument"
345 )));
346 }
347 let histograms = extract_histogram_array(&input[0], name)?;
348 let histograms = histograms
349 .as_any()
350 .downcast_ref::<StructArray>()
351 .expect("validated native histogram struct");
352 let mut result = Float64Builder::with_capacity(histograms.len());
353 for row in 0..histograms.len() {
354 match read_histogram(histograms, row)? {
355 Some(histogram) => {
356 result.append_value(calc(&histogram, input, row, histograms.len(), name)?)
357 }
358 None => result.append_null(),
359 }
360 }
361 Ok(ColumnarValue::Array(Arc::new(result.finish())))
362 }) as _,
363 )
364}
365
366fn histogram_pair_udf(
367 name: &'static str,
368 op: fn(&NativeHistogram, &NativeHistogram) -> Option<NativeHistogram>,
369) -> ScalarUDF {
370 histogram_pair_udf_with_collector(name, op, None)
371}
372
373fn histogram_pair_udf_with_collector(
374 name: &'static str,
375 op: fn(&NativeHistogram, &NativeHistogram) -> Option<NativeHistogram>,
376 collector: Option<PromqlAnnotationCollector>,
377) -> ScalarUDF {
378 create_udf(
379 name,
380 vec![native_histogram_arrow_type(), native_histogram_arrow_type()],
381 native_histogram_arrow_type(),
382 Volatility::Volatile,
383 Arc::new(move |input: &[ColumnarValue]| {
384 let lhs = extract_histogram_array(&input[0], name)?;
385 let rhs = extract_histogram_array(&input[1], name)?;
386 if lhs.len() != rhs.len() {
387 return Err(DataFusionError::Execution(format!(
388 "{name}: native histogram argument length mismatch: {} vs {}",
389 lhs.len(),
390 rhs.len()
391 )));
392 }
393
394 let lhs = lhs
395 .as_any()
396 .downcast_ref::<StructArray>()
397 .expect("validated native histogram struct");
398 let rhs = rhs
399 .as_any()
400 .downcast_ref::<StructArray>()
401 .expect("validated native histogram struct");
402 let mut result = Vec::with_capacity(lhs.len());
403 for row in 0..lhs.len() {
404 result.push(
405 match (read_histogram(lhs, row)?, read_histogram(rhs, row)?) {
406 (Some(lhs), Some(rhs)) => {
407 record_custom_reconciliation(&collector, name, &lhs, &rhs);
408 record_counter_reset_contradiction(&collector, name, &lhs, &rhs);
409 let result = op(&lhs, &rhs);
410 if result.is_none() {
411 record_warning(
412 &collector,
413 format!(
414 "{name}: dropped native histogram sample with incompatible schemas"
415 ),
416 );
417 }
418 result
419 }
420 _ => None,
421 },
422 );
423 }
424 Ok(ColumnarValue::Array(build_histogram_array(&result)))
425 }) as _,
426 )
427}
428
429fn histogram_transform_udf(
430 name: &'static str,
431 op: fn(NativeHistogram) -> NativeHistogram,
432) -> ScalarUDF {
433 create_udf(
434 name,
435 vec![native_histogram_arrow_type()],
436 native_histogram_arrow_type(),
437 Volatility::Volatile,
438 Arc::new(move |input: &[ColumnarValue]| {
439 let histograms = extract_histogram_array(&input[0], name)?;
440 let histograms = histograms
441 .as_any()
442 .downcast_ref::<StructArray>()
443 .expect("validated native histogram struct");
444 let mut result = Vec::with_capacity(histograms.len());
445 for row in 0..histograms.len() {
446 result.push(read_histogram(histograms, row)?.map(op));
447 }
448 Ok(ColumnarValue::Array(build_histogram_array(&result)))
449 }) as _,
450 )
451}
452
453fn histogram_string_udf(name: &'static str) -> ScalarUDF {
454 create_udf(
455 name,
456 vec![native_histogram_arrow_type()],
457 DataType::Utf8,
458 Volatility::Volatile,
459 Arc::new(move |input: &[ColumnarValue]| {
460 let histograms = extract_histogram_array(&input[0], name)?;
461 let histograms = histograms
462 .as_any()
463 .downcast_ref::<StructArray>()
464 .expect("validated native histogram struct");
465 let mut result = StringBuilder::with_capacity(histograms.len(), histograms.len() * 32);
466 for row in 0..histograms.len() {
467 match read_histogram(histograms, row)? {
468 Some(histogram) => result.append_value(histogram.promql_string()),
469 None => result.append_null(),
470 }
471 }
472 Ok(ColumnarValue::Array(Arc::new(result.finish())))
473 }) as _,
474 )
475}
476
477fn histogram_scalar_udf(
478 name: &'static str,
479 input_types: Vec<DataType>,
480 histogram_index: usize,
481 scalar_index: usize,
482 op: fn(NativeHistogram, f64) -> Option<NativeHistogram>,
483) -> ScalarUDF {
484 create_udf(
485 name,
486 input_types,
487 native_histogram_arrow_type(),
488 Volatility::Volatile,
489 Arc::new(move |input: &[ColumnarValue]| {
490 let histograms = extract_histogram_array(&input[histogram_index], name)?;
491 let histograms = histograms
492 .as_any()
493 .downcast_ref::<StructArray>()
494 .expect("validated native histogram struct");
495 let mut result = Vec::with_capacity(histograms.len());
496 for row in 0..histograms.len() {
497 result.push(match read_histogram(histograms, row)? {
498 Some(histogram) => {
499 let scalar =
500 read_scalar_f64_arg(&input[scalar_index], row, histograms.len(), name)?;
501 op(histogram, scalar)
502 }
503 None => None,
504 });
505 }
506 Ok(ColumnarValue::Array(build_histogram_array(&result)))
507 }) as _,
508 )
509}
510
511fn histogram_compare_udf(
512 name: &'static str,
513 op: fn(&NativeHistogram, &NativeHistogram) -> bool,
514) -> ScalarUDF {
515 create_udf(
516 name,
517 vec![native_histogram_arrow_type(), native_histogram_arrow_type()],
518 DataType::Boolean,
519 Volatility::Volatile,
520 Arc::new(move |input: &[ColumnarValue]| {
521 let lhs = extract_histogram_array(&input[0], name)?;
522 let rhs = extract_histogram_array(&input[1], name)?;
523 if lhs.len() != rhs.len() {
524 return Err(DataFusionError::Execution(format!(
525 "{name}: native histogram argument length mismatch: {} vs {}",
526 lhs.len(),
527 rhs.len()
528 )));
529 }
530
531 let lhs = lhs
532 .as_any()
533 .downcast_ref::<StructArray>()
534 .expect("validated native histogram struct");
535 let rhs = rhs
536 .as_any()
537 .downcast_ref::<StructArray>()
538 .expect("validated native histogram struct");
539 let mut result = Vec::with_capacity(lhs.len());
540 for row in 0..lhs.len() {
541 result.push(
542 match (read_histogram(lhs, row)?, read_histogram(rhs, row)?) {
543 (Some(lhs), Some(rhs)) => Some(op(&lhs, &rhs)),
544 _ => None,
545 },
546 );
547 }
548 Ok(ColumnarValue::Array(Arc::new(BooleanArray::from(result))))
549 }) as _,
550 )
551}
552
553pub struct NativeHistogramAdd;
554
555impl NativeHistogramAdd {
556 pub const fn name() -> &'static str {
557 "prom_native_histogram_add"
558 }
559
560 pub fn scalar_udf() -> ScalarUDF {
561 histogram_pair_udf(Self::name(), NativeHistogram::add)
562 }
563
564 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
565 histogram_pair_udf_with_collector(Self::name(), NativeHistogram::add, collector)
566 }
567}
568
569pub struct NativeHistogramSub;
570
571impl NativeHistogramSub {
572 pub const fn name() -> &'static str {
573 "prom_native_histogram_sub"
574 }
575
576 pub fn scalar_udf() -> ScalarUDF {
577 histogram_pair_udf(Self::name(), NativeHistogram::sub)
578 }
579
580 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
581 histogram_pair_udf_with_collector(Self::name(), NativeHistogram::sub, collector)
582 }
583}
584
585pub struct NativeHistogramMulScalar;
586
587impl NativeHistogramMulScalar {
588 pub const fn name() -> &'static str {
589 "prom_native_histogram_mul_scalar"
590 }
591
592 pub fn scalar_udf() -> ScalarUDF {
593 histogram_scalar_udf(
594 Self::name(),
595 vec![native_histogram_arrow_type(), DataType::Float64],
596 0,
597 1,
598 |histogram, scalar| Some(histogram.scale(scalar)),
599 )
600 }
601}
602
603pub struct NativeHistogramScalarMul;
604
605impl NativeHistogramScalarMul {
606 pub const fn name() -> &'static str {
607 "prom_native_histogram_scalar_mul"
608 }
609
610 pub fn scalar_udf() -> ScalarUDF {
611 histogram_scalar_udf(
612 Self::name(),
613 vec![DataType::Float64, native_histogram_arrow_type()],
614 1,
615 0,
616 |histogram, scalar| Some(histogram.scale(scalar)),
617 )
618 }
619}
620
621pub struct NativeHistogramDivScalar;
622
623impl NativeHistogramDivScalar {
624 pub const fn name() -> &'static str {
625 "prom_native_histogram_div_scalar"
626 }
627
628 pub fn scalar_udf() -> ScalarUDF {
629 histogram_scalar_udf(
630 Self::name(),
631 vec![native_histogram_arrow_type(), DataType::Float64],
632 0,
633 1,
634 |histogram, scalar| Some(histogram.divide_by(scalar)),
635 )
636 }
637}
638
639pub struct NativeHistogramNeg;
640
641impl NativeHistogramNeg {
642 pub const fn name() -> &'static str {
643 "prom_native_histogram_neg"
644 }
645
646 pub fn scalar_udf() -> ScalarUDF {
647 histogram_transform_udf(Self::name(), NativeHistogram::negated)
648 }
649}
650
651pub struct NativeHistogramEq;
652
653impl NativeHistogramEq {
654 pub const fn name() -> &'static str {
655 "prom_native_histogram_eq"
656 }
657
658 pub fn scalar_udf() -> ScalarUDF {
659 histogram_compare_udf(Self::name(), NativeHistogram::promql_eq)
660 }
661}
662
663pub struct NativeHistogramNotEq;
664
665impl NativeHistogramNotEq {
666 pub const fn name() -> &'static str {
667 "prom_native_histogram_not_eq"
668 }
669
670 pub fn scalar_udf() -> ScalarUDF {
671 histogram_compare_udf(Self::name(), |lhs, rhs| !lhs.promql_eq(rhs))
672 }
673}
674
675pub struct NativeHistogramCount;
676
677impl NativeHistogramCount {
678 pub const fn name() -> &'static str {
679 "prom_native_histogram_count"
680 }
681
682 pub fn scalar_udf() -> ScalarUDF {
683 scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
684 Ok(histogram.count)
685 })
686 }
687}
688
689pub struct NativeHistogramSum;
690
691impl NativeHistogramSum {
692 pub const fn name() -> &'static str {
693 "prom_native_histogram_sum"
694 }
695
696 pub fn scalar_udf() -> ScalarUDF {
697 scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
698 Ok(histogram.sum)
699 })
700 }
701}
702
703pub struct NativeHistogramAvg;
704
705impl NativeHistogramAvg {
706 pub const fn name() -> &'static str {
707 "prom_native_histogram_avg"
708 }
709
710 pub fn scalar_udf() -> ScalarUDF {
711 scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
712 Ok(histogram.sum / histogram.count)
713 })
714 }
715}
716
717pub struct NativeHistogramStddev;
718
719impl NativeHistogramStddev {
720 pub const fn name() -> &'static str {
721 "prom_native_histogram_stddev"
722 }
723
724 pub fn scalar_udf() -> ScalarUDF {
725 scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
726 Ok(histogram.estimated_stddev())
727 })
728 }
729}
730
731pub struct NativeHistogramStdvar;
732
733impl NativeHistogramStdvar {
734 pub const fn name() -> &'static str {
735 "prom_native_histogram_stdvar"
736 }
737
738 pub fn scalar_udf() -> ScalarUDF {
739 scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
740 Ok(histogram.estimated_stdvar())
741 })
742 }
743}
744
745pub struct PromqlFloatToString;
747
748impl PromqlFloatToString {
749 pub const fn name() -> &'static str {
750 "prom_float_to_string"
751 }
752
753 pub fn scalar_udf() -> ScalarUDF {
754 create_udf(
755 Self::name(),
756 vec![DataType::Float64],
757 DataType::Utf8,
758 Volatility::Volatile,
759 Arc::new(|input: &[ColumnarValue]| {
760 let values = extract_array(&input[0])?;
761 let values = values
762 .as_any()
763 .downcast_ref::<Float64Array>()
764 .expect("validated Float64 input");
765 let mut result = StringBuilder::new();
766 for value in values.iter() {
767 match value {
768 Some(value) => result.append_value(format_prometheus_float(value)),
769 None => result.append_null(),
770 }
771 }
772 Ok(ColumnarValue::Array(Arc::new(result.finish())))
773 }),
774 )
775 }
776}
777
778pub struct NativeHistogramToString;
779
780impl NativeHistogramToString {
781 pub const fn name() -> &'static str {
782 "prom_native_histogram_to_string"
783 }
784
785 pub fn scalar_udf() -> ScalarUDF {
786 histogram_string_udf(Self::name())
787 }
788}
789
790pub struct NativeHistogramQuantile;
791
792impl NativeHistogramQuantile {
793 pub const fn name() -> &'static str {
794 "prom_native_histogram_quantile"
795 }
796
797 pub fn scalar_udf() -> ScalarUDF {
798 scalar_histogram_udf(
799 Self::name(),
800 vec![DataType::Float64],
801 |histogram, input, row, len, name| {
802 Ok(histogram.quantile(read_scalar_f64_arg(&input[1], row, len, name)?))
803 },
804 )
805 }
806}
807
808pub struct NativeHistogramFraction;
809
810impl NativeHistogramFraction {
811 pub const fn name() -> &'static str {
812 "prom_native_histogram_fraction"
813 }
814
815 pub fn scalar_udf() -> ScalarUDF {
816 scalar_histogram_udf(
817 Self::name(),
818 vec![DataType::Float64, DataType::Float64],
819 |histogram, input, row, len, name| {
820 let lower = read_scalar_f64_arg(&input[1], row, len, name)?;
821 let upper = read_scalar_f64_arg(&input[2], row, len, name)?;
822 Ok(histogram.fraction(lower, upper))
823 },
824 )
825 }
826}
827
828#[derive(Debug, Clone, Copy)]
829enum NativeHistogramAggregateKind {
830 Sum,
831 Avg,
832}
833
834impl NativeHistogramAggregateKind {
835 const fn name(self) -> &'static str {
836 match self {
837 Self::Sum => NativeHistogramAggSum::name(),
838 Self::Avg => NativeHistogramAggAvg::name(),
839 }
840 }
841
842 const fn needs_count(self) -> bool {
843 matches!(self, Self::Avg)
844 }
845}
846
847pub struct NativeHistogramAggSum;
848
849impl NativeHistogramAggSum {
850 pub const fn name() -> &'static str {
851 "prom_native_histogram_agg_sum"
852 }
853
854 pub fn aggregate_udf() -> AggregateUDF {
855 native_histogram_aggregate_udf(NativeHistogramAggregateKind::Sum, None)
856 }
857
858 pub fn aggregate_udf_with_collector(
859 collector: Option<PromqlAnnotationCollector>,
860 ) -> AggregateUDF {
861 native_histogram_aggregate_udf(NativeHistogramAggregateKind::Sum, collector)
862 }
863}
864
865pub struct NativeHistogramAggAvg;
866
867impl NativeHistogramAggAvg {
868 pub const fn name() -> &'static str {
869 "prom_native_histogram_agg_avg"
870 }
871
872 pub fn aggregate_udf() -> AggregateUDF {
873 native_histogram_aggregate_udf(NativeHistogramAggregateKind::Avg, None)
874 }
875
876 pub fn aggregate_udf_with_collector(
877 collector: Option<PromqlAnnotationCollector>,
878 ) -> AggregateUDF {
879 native_histogram_aggregate_udf(NativeHistogramAggregateKind::Avg, collector)
880 }
881}
882
883#[derive(Debug)]
884struct NativeHistogramAggregateAccumulator {
885 kind: NativeHistogramAggregateKind,
886 value: Option<NativeHistogram>,
887 count: u64,
888 dropped_incompatible: bool,
889 counter_reset_seen: bool,
890 not_counter_reset_seen: bool,
891 collector: Option<PromqlAnnotationCollector>,
892}
893
894impl NativeHistogramAggregateAccumulator {
895 fn new(
896 kind: NativeHistogramAggregateKind,
897 collector: Option<PromqlAnnotationCollector>,
898 ) -> Self {
899 Self {
900 kind,
901 value: None,
902 count: 0,
903 dropped_incompatible: false,
904 counter_reset_seen: false,
905 not_counter_reset_seen: false,
906 collector,
907 }
908 }
909
910 fn from_args(
911 kind: NativeHistogramAggregateKind,
912 collector: Option<PromqlAnnotationCollector>,
913 _args: AccumulatorArgs,
914 ) -> DfResult<Box<dyn DfAccumulator>> {
915 Ok(Box::new(Self::new(kind, collector)))
916 }
917
918 fn observe_reset_hints(&mut self, counter_reset_seen: bool, not_counter_reset_seen: bool) {
919 self.counter_reset_seen |= counter_reset_seen;
920 self.not_counter_reset_seen |= not_counter_reset_seen;
921 if self.counter_reset_seen && self.not_counter_reset_seen {
922 record_counter_reset_contradiction_warning(&self.collector, self.kind.name());
923 }
924 }
925
926 fn push_histogram(&mut self, histogram: NativeHistogram, count: u64) -> DfResult<()> {
927 if self.kind.needs_count() && count == 0 {
928 return Ok(());
929 }
930
931 self.observe_reset_hints(
932 histogram.reset_hint == COUNTER_RESET_HINT,
933 histogram.reset_hint == NOT_COUNTER_RESET_HINT,
934 );
935 if self.dropped_incompatible {
936 return Ok(());
937 }
938 let combined_count = if self.kind.needs_count() {
939 self.count.checked_add(count).ok_or_else(|| {
940 DataFusionError::Execution(format!(
941 "{}: native histogram sample count overflow",
942 self.kind.name()
943 ))
944 })?
945 } else {
946 self.count
947 };
948 let value = match self.value.take() {
949 Some(value) => {
950 record_custom_reconciliation(&self.collector, self.kind.name(), &value, &histogram);
951 let combined = match self.kind {
952 NativeHistogramAggregateKind::Sum => value.add(&histogram),
953 NativeHistogramAggregateKind::Avg => {
954 weighted_histogram_mean(value, self.count, histogram, count, combined_count)
955 }
956 };
957 match combined {
958 Some(value) => Some(value),
959 None => {
960 self.record_incompatible();
961 None
962 }
963 }
964 }
965 None => Some(histogram),
966 };
967 if !self.dropped_incompatible {
968 self.value = value;
969 self.count = combined_count;
970 }
971 Ok(())
972 }
973
974 fn mark_incompatible(&mut self) {
975 self.value = None;
976 self.count = 0;
977 self.dropped_incompatible = true;
978 }
979
980 fn record_incompatible(&mut self) {
981 self.mark_incompatible();
982 record_warning(
983 &self.collector,
984 format!(
985 "{}: dropped native histogram aggregate with incompatible schemas",
986 self.kind.name()
987 ),
988 );
989 }
990}
991
992fn weighted_histogram_mean(
993 left: NativeHistogram,
994 left_count: u64,
995 right: NativeHistogram,
996 right_count: u64,
997 total_count: u64,
998) -> Option<NativeHistogram> {
999 let total_count = total_count as f64;
1000 left.scale(left_count as f64 / total_count)
1001 .add(&right.scale(right_count as f64 / total_count))
1002}
1003
1004fn range_fold_histograms(
1005 samples: Vec<NativeHistogram>,
1006 kind: NativeHistogramAggregateKind,
1007 name: &'static str,
1008 collector: &Option<PromqlAnnotationCollector>,
1009) -> Option<NativeHistogram> {
1010 if samples
1011 .iter()
1012 .any(|histogram| histogram.reset_hint == COUNTER_RESET_HINT)
1013 && samples
1014 .iter()
1015 .any(|histogram| histogram.reset_hint == NOT_COUNTER_RESET_HINT)
1016 {
1017 record_counter_reset_contradiction_warning(collector, name);
1018 }
1019
1020 let mut value = None;
1021 let mut count = 0u64;
1022 for histogram in samples {
1023 value = match value {
1024 Some(value) => {
1025 record_custom_reconciliation(collector, name, &value, &histogram);
1026 let next_count = count.checked_add(1)?;
1027 let combined = match kind {
1028 NativeHistogramAggregateKind::Sum => value.add(&histogram),
1029 NativeHistogramAggregateKind::Avg => {
1030 weighted_histogram_mean(value, count, histogram, 1, next_count)
1031 }
1032 };
1033 match combined {
1034 Some(value) => Some(value),
1035 None => {
1036 record_warning(
1037 collector,
1038 format!(
1039 "{name}: dropped native histogram range with incompatible schemas"
1040 ),
1041 );
1042 return None;
1043 }
1044 }
1045 }
1046 None => Some(histogram),
1047 };
1048 count = count.checked_add(1)?;
1049 }
1050 value
1051}
1052
1053#[derive(Debug, Clone, Copy)]
1054enum NativeHistogramRangeHistogramKind {
1055 Sum,
1056 Avg,
1057 Last,
1058}
1059
1060#[derive(Debug, Clone, Copy)]
1061enum NativeHistogramRangeFloatKind {
1062 Absent,
1063 Count,
1064 Present,
1065 Changes,
1066 Resets,
1067}
1068
1069fn collect_window_histograms(
1070 histograms: &StructArray,
1071 offset: usize,
1072 length: usize,
1073) -> DfResult<Option<Vec<NativeHistogram>>> {
1074 let mut samples = Vec::with_capacity(length);
1075 for row in offset..offset + length {
1076 let Some(histogram) = read_histogram(histograms, row)? else {
1077 return Ok(None);
1078 };
1079 samples.push(histogram);
1080 }
1081 Ok(Some(samples))
1082}
1083
1084fn native_histogram_range_histogram(
1085 input: &[ColumnarValue],
1086 kind: NativeHistogramRangeHistogramKind,
1087 func_name: &'static str,
1088 collector: Option<PromqlAnnotationCollector>,
1089) -> DfResult<ColumnarValue> {
1090 if input.len() != 2 {
1091 return Err(DataFusionError::Plan(format!(
1092 "{func_name} function should have 2 inputs"
1093 )));
1094 }
1095
1096 let ts_range = extract_range_dict(
1097 &input[0],
1098 func_name,
1099 "timestamp range vector",
1100 &DataType::Timestamp(TimeUnit::Millisecond, None),
1101 )?;
1102 let value_range = extract_range_dict(
1103 &input[1],
1104 func_name,
1105 "value range vector",
1106 &native_histogram_arrow_type(),
1107 )?;
1108 if ts_range.keys().values() != value_range.keys().values() {
1109 return Err(DataFusionError::Execution(format!(
1110 "{func_name}: timestamp and value ranges should have the same window layout"
1111 )));
1112 }
1113
1114 let histograms = value_range
1115 .values()
1116 .as_any()
1117 .downcast_ref::<StructArray>()
1118 .expect("validated native histogram range");
1119 let mut result = Vec::with_capacity(value_range.keys().len());
1120 for key in value_range.keys().values() {
1121 let (offset, length) = unpack(*key);
1122 let offset = offset as usize;
1123 let length = length as usize;
1124 if length == 0 {
1125 result.push(None);
1126 continue;
1127 }
1128 if matches!(kind, NativeHistogramRangeHistogramKind::Last) {
1129 let histogram = if (offset..offset + length).any(|row| histograms.is_null(row)) {
1130 None
1131 } else {
1132 read_histogram(histograms, offset + length - 1)?
1133 };
1134 result.push(histogram);
1135 continue;
1136 }
1137 let Some(samples) = collect_window_histograms(histograms, offset, length)? else {
1138 result.push(None);
1139 continue;
1140 };
1141 let histogram = match kind {
1142 NativeHistogramRangeHistogramKind::Sum => range_fold_histograms(
1143 samples,
1144 NativeHistogramAggregateKind::Sum,
1145 func_name,
1146 &collector,
1147 ),
1148 NativeHistogramRangeHistogramKind::Avg => range_fold_histograms(
1149 samples,
1150 NativeHistogramAggregateKind::Avg,
1151 func_name,
1152 &collector,
1153 ),
1154 NativeHistogramRangeHistogramKind::Last => samples.last().cloned(),
1155 };
1156 result.push(histogram);
1157 }
1158
1159 Ok(ColumnarValue::Array(build_histogram_array(&result)))
1160}
1161
1162fn native_histogram_range_float(
1163 input: &[ColumnarValue],
1164 kind: NativeHistogramRangeFloatKind,
1165 func_name: &'static str,
1166) -> DfResult<ColumnarValue> {
1167 if input.len() != 2 {
1168 return Err(DataFusionError::Plan(format!(
1169 "{func_name} function should have 2 inputs"
1170 )));
1171 }
1172
1173 let ts_range = extract_range_dict(
1174 &input[0],
1175 func_name,
1176 "timestamp range vector",
1177 &DataType::Timestamp(TimeUnit::Millisecond, None),
1178 )?;
1179 let value_range = extract_range_dict(
1180 &input[1],
1181 func_name,
1182 "value range vector",
1183 &native_histogram_arrow_type(),
1184 )?;
1185 if ts_range.keys().values() != value_range.keys().values() {
1186 return Err(DataFusionError::Execution(format!(
1187 "{func_name}: timestamp and value ranges should have the same window layout"
1188 )));
1189 }
1190
1191 let timestamps = ts_range
1192 .values()
1193 .as_any()
1194 .downcast_ref::<TimestampMillisecondArray>()
1195 .expect("validated timestamp range")
1196 .values();
1197 let histograms = value_range
1198 .values()
1199 .as_any()
1200 .downcast_ref::<StructArray>()
1201 .expect("validated native histogram range");
1202 let mut result = Float64Builder::with_capacity(value_range.keys().len());
1203 for key in value_range.keys().values() {
1204 let (offset, length) = unpack(*key);
1205 let offset = offset as usize;
1206 let length = length as usize;
1207 if length == 0 {
1208 match kind {
1209 NativeHistogramRangeFloatKind::Absent => result.append_value(1.0),
1210 _ => result.append_null(),
1211 }
1212 continue;
1213 }
1214 if matches!(kind, NativeHistogramRangeFloatKind::Absent) {
1215 result.append_null();
1216 continue;
1217 }
1218 if matches!(
1219 kind,
1220 NativeHistogramRangeFloatKind::Count | NativeHistogramRangeFloatKind::Present
1221 ) {
1222 if (offset..offset + length).any(|row| histograms.is_null(row)) {
1223 result.append_null();
1224 } else if matches!(kind, NativeHistogramRangeFloatKind::Count) {
1225 result.append_value(length as f64);
1226 } else {
1227 result.append_value(1.0);
1228 }
1229 continue;
1230 }
1231 let Some(samples) = collect_window_histograms(histograms, offset, length)? else {
1232 result.append_null();
1233 continue;
1234 };
1235 let value = match kind {
1236 NativeHistogramRangeFloatKind::Absent => {
1237 result.append_null();
1238 continue;
1239 }
1240 NativeHistogramRangeFloatKind::Count => length as f64,
1241 NativeHistogramRangeFloatKind::Present => 1.0,
1242 NativeHistogramRangeFloatKind::Changes => samples
1243 .windows(2)
1244 .filter(|pair| !pair[0].promql_eq(&pair[1]))
1245 .count() as f64,
1246 NativeHistogramRangeFloatKind::Resets => samples
1247 .windows(2)
1248 .zip(timestamps[offset..offset + length].windows(2))
1249 .filter(|(pair, ts_pair)| {
1250 (pair[0].reset_hint == GAUGE_RESET_HINT)
1251 != (pair[1].reset_hint == GAUGE_RESET_HINT)
1252 || pair[1].detect_counter_reset(&pair[0], ts_pair[0], ts_pair[1])
1253 })
1254 .count() as f64,
1255 };
1256 result.append_value(value);
1257 }
1258
1259 Ok(ColumnarValue::Array(Arc::new(result.finish())))
1260}
1261
1262fn create_native_range_histogram_udf(
1263 name: &'static str,
1264 kind: NativeHistogramRangeHistogramKind,
1265 collector: Option<PromqlAnnotationCollector>,
1266) -> ScalarUDF {
1267 create_udf(
1268 name,
1269 vec![
1270 RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
1271 RangeArray::convert_data_type(native_histogram_arrow_type()),
1272 ],
1273 native_histogram_arrow_type(),
1274 Volatility::Volatile,
1275 Arc::new(move |input: &[ColumnarValue]| {
1276 native_histogram_range_histogram(input, kind, name, collector.clone())
1277 }) as _,
1278 )
1279}
1280
1281fn create_native_range_float_udf(
1282 name: &'static str,
1283 kind: NativeHistogramRangeFloatKind,
1284) -> ScalarUDF {
1285 create_udf(
1286 name,
1287 vec![
1288 RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
1289 RangeArray::convert_data_type(native_histogram_arrow_type()),
1290 ],
1291 DataType::Float64,
1292 Volatility::Volatile,
1293 Arc::new(move |input: &[ColumnarValue]| native_histogram_range_float(input, kind, name))
1294 as _,
1295 )
1296}
1297
1298pub struct NativeHistogramSumOverTime;
1299pub struct NativeHistogramAvgOverTime;
1300pub struct NativeHistogramAbsentOverTime;
1301pub struct NativeHistogramCountOverTime;
1302pub struct NativeHistogramLastOverTime;
1303pub struct NativeHistogramPresentOverTime;
1304pub struct NativeHistogramChanges;
1305pub struct NativeHistogramResets;
1306
1307impl NativeHistogramSumOverTime {
1308 pub const fn name() -> &'static str {
1309 "prom_native_histogram_sum_over_time"
1310 }
1311
1312 pub fn scalar_udf() -> ScalarUDF {
1313 Self::scalar_udf_with_collector(None)
1314 }
1315
1316 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1317 create_native_range_histogram_udf(
1318 Self::name(),
1319 NativeHistogramRangeHistogramKind::Sum,
1320 collector,
1321 )
1322 }
1323}
1324
1325impl NativeHistogramAvgOverTime {
1326 pub const fn name() -> &'static str {
1327 "prom_native_histogram_avg_over_time"
1328 }
1329
1330 pub fn scalar_udf() -> ScalarUDF {
1331 Self::scalar_udf_with_collector(None)
1332 }
1333
1334 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1335 create_native_range_histogram_udf(
1336 Self::name(),
1337 NativeHistogramRangeHistogramKind::Avg,
1338 collector,
1339 )
1340 }
1341}
1342
1343impl NativeHistogramAbsentOverTime {
1344 pub const fn name() -> &'static str {
1345 "prom_native_histogram_absent_over_time"
1346 }
1347
1348 pub fn scalar_udf() -> ScalarUDF {
1349 create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Absent)
1350 }
1351}
1352
1353impl NativeHistogramCountOverTime {
1354 pub const fn name() -> &'static str {
1355 "prom_native_histogram_count_over_time"
1356 }
1357
1358 pub fn scalar_udf() -> ScalarUDF {
1359 create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Count)
1360 }
1361}
1362
1363impl NativeHistogramLastOverTime {
1364 pub const fn name() -> &'static str {
1365 "prom_native_histogram_last_over_time"
1366 }
1367
1368 pub fn scalar_udf() -> ScalarUDF {
1369 create_native_range_histogram_udf(
1370 Self::name(),
1371 NativeHistogramRangeHistogramKind::Last,
1372 None,
1373 )
1374 }
1375}
1376
1377impl NativeHistogramPresentOverTime {
1378 pub const fn name() -> &'static str {
1379 "prom_native_histogram_present_over_time"
1380 }
1381
1382 pub fn scalar_udf() -> ScalarUDF {
1383 create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Present)
1384 }
1385}
1386
1387impl NativeHistogramChanges {
1388 pub const fn name() -> &'static str {
1389 "prom_native_histogram_changes"
1390 }
1391
1392 pub fn scalar_udf() -> ScalarUDF {
1393 create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Changes)
1394 }
1395}
1396
1397impl NativeHistogramResets {
1398 pub const fn name() -> &'static str {
1399 "prom_native_histogram_resets"
1400 }
1401
1402 pub fn scalar_udf() -> ScalarUDF {
1403 create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Resets)
1404 }
1405}
1406
1407pub struct MixedRange;
1413
1414impl MixedRange {
1415 const fn float_name() -> &'static str {
1416 "prom_mixed_range_float"
1417 }
1418
1419 const fn histogram_name() -> &'static str {
1420 "prom_mixed_range_histogram"
1421 }
1422
1423 pub fn float_udf(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1424 ScalarUDF::new_from_impl(MixedRangeUdf::new(MixedRangeOutput::Float, collector))
1425 }
1426
1427 pub fn histogram_udf(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1428 ScalarUDF::new_from_impl(MixedRangeUdf::new(MixedRangeOutput::Histogram, collector))
1429 }
1430}
1431
1432#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1433enum MixedRangeOutput {
1434 Float,
1435 Histogram,
1436}
1437
1438impl MixedRangeOutput {
1439 fn name(self) -> &'static str {
1440 match self {
1441 Self::Float => MixedRange::float_name(),
1442 Self::Histogram => MixedRange::histogram_name(),
1443 }
1444 }
1445
1446 fn data_type(self) -> DataType {
1447 match self {
1448 Self::Float => DataType::Float64,
1449 Self::Histogram => native_histogram_arrow_type(),
1450 }
1451 }
1452}
1453
1454#[derive(Debug, Clone)]
1455struct MixedRangeUdf {
1456 output: MixedRangeOutput,
1457 signature: Signature,
1458 collector: Option<PromqlAnnotationCollector>,
1459}
1460
1461impl MixedRangeUdf {
1462 fn new(output: MixedRangeOutput, collector: Option<PromqlAnnotationCollector>) -> Self {
1463 Self {
1464 output,
1465 signature: Signature::variadic_any(Volatility::Volatile),
1466 collector,
1467 }
1468 }
1469}
1470
1471impl PartialEq for MixedRangeUdf {
1472 fn eq(&self, other: &Self) -> bool {
1473 self.output == other.output
1474 }
1475}
1476
1477impl Eq for MixedRangeUdf {}
1478
1479impl Hash for MixedRangeUdf {
1480 fn hash<H: Hasher>(&self, state: &mut H) {
1481 self.output.hash(state);
1482 }
1483}
1484
1485impl ScalarUDFImpl for MixedRangeUdf {
1486 fn as_any(&self) -> &dyn Any {
1487 self
1488 }
1489
1490 fn name(&self) -> &str {
1491 self.output.name()
1492 }
1493
1494 fn signature(&self) -> &Signature {
1495 &self.signature
1496 }
1497
1498 fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
1499 Ok(self.output.data_type())
1500 }
1501
1502 fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult<ColumnarValue> {
1503 let collector = args
1504 .config_options
1505 .extensions
1506 .get::<PromqlAnnotationCollector>()
1507 .cloned()
1508 .or_else(|| self.collector.clone());
1509 mixed_range(&args, self.output, collector)
1510 }
1511}
1512
1513#[derive(Debug, Clone, Copy)]
1514enum MixedRangeFunction {
1515 Rate,
1516 Increase,
1517 Delta,
1518 IDelta,
1519 IRate,
1520 Changes,
1521 Resets,
1522 AvgOverTime,
1523 MinOverTime,
1524 MaxOverTime,
1525 SumOverTime,
1526 CountOverTime,
1527 LastOverTime,
1528 AbsentOverTime,
1529 PresentOverTime,
1530 StddevOverTime,
1531 StdvarOverTime,
1532 QuantileOverTime,
1533 Deriv,
1534 PredictLinear,
1535 DoubleExponentialSmoothing,
1536 HoltWinters,
1537}
1538
1539impl MixedRangeFunction {
1540 fn parse(value: &ColumnarValue) -> DfResult<Self> {
1541 let name = match value {
1542 ColumnarValue::Scalar(ScalarValue::Utf8(Some(name)))
1543 | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(name)))
1544 | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(name))) => name.as_str(),
1545 other => {
1546 return Err(DataFusionError::Execution(format!(
1547 "mixed range function name must be a non-null string scalar, found {}",
1548 other.data_type()
1549 )));
1550 }
1551 };
1552
1553 match name {
1554 "rate" => Ok(Self::Rate),
1555 "increase" => Ok(Self::Increase),
1556 "delta" => Ok(Self::Delta),
1557 "idelta" => Ok(Self::IDelta),
1558 "irate" => Ok(Self::IRate),
1559 "changes" => Ok(Self::Changes),
1560 "resets" => Ok(Self::Resets),
1561 "avg_over_time" => Ok(Self::AvgOverTime),
1562 "min_over_time" => Ok(Self::MinOverTime),
1563 "max_over_time" => Ok(Self::MaxOverTime),
1564 "sum_over_time" => Ok(Self::SumOverTime),
1565 "count_over_time" => Ok(Self::CountOverTime),
1566 "last_over_time" => Ok(Self::LastOverTime),
1567 "absent_over_time" => Ok(Self::AbsentOverTime),
1568 "present_over_time" => Ok(Self::PresentOverTime),
1569 "stddev_over_time" => Ok(Self::StddevOverTime),
1570 "stdvar_over_time" => Ok(Self::StdvarOverTime),
1571 "quantile_over_time" => Ok(Self::QuantileOverTime),
1572 "deriv" => Ok(Self::Deriv),
1573 "predict_linear" => Ok(Self::PredictLinear),
1574 "double_exponential_smoothing" => Ok(Self::DoubleExponentialSmoothing),
1575 "holt_winters" => Ok(Self::HoltWinters),
1576 _ => Err(DataFusionError::Execution(format!(
1577 "unsupported mixed range function: {name}"
1578 ))),
1579 }
1580 }
1581
1582 fn name(self) -> &'static str {
1583 match self {
1584 Self::Rate => "rate",
1585 Self::Increase => "increase",
1586 Self::Delta => "delta",
1587 Self::IDelta => "idelta",
1588 Self::IRate => "irate",
1589 Self::Changes => "changes",
1590 Self::Resets => "resets",
1591 Self::AvgOverTime => "avg_over_time",
1592 Self::MinOverTime => "min_over_time",
1593 Self::MaxOverTime => "max_over_time",
1594 Self::SumOverTime => "sum_over_time",
1595 Self::CountOverTime => "count_over_time",
1596 Self::LastOverTime => "last_over_time",
1597 Self::AbsentOverTime => "absent_over_time",
1598 Self::PresentOverTime => "present_over_time",
1599 Self::StddevOverTime => "stddev_over_time",
1600 Self::StdvarOverTime => "stdvar_over_time",
1601 Self::QuantileOverTime => "quantile_over_time",
1602 Self::Deriv => "deriv",
1603 Self::PredictLinear => "predict_linear",
1604 Self::DoubleExponentialSmoothing => "double_exponential_smoothing",
1605 Self::HoltWinters => "holt_winters",
1606 }
1607 }
1608
1609 fn policy(self) -> MixedRangePolicy {
1610 match self {
1611 Self::Rate | Self::Increase | Self::Delta | Self::AvgOverTime | Self::SumOverTime => {
1612 MixedRangePolicy::DropMixed
1613 }
1614 Self::IDelta | Self::IRate => MixedRangePolicy::LastTwo,
1615 Self::LastOverTime => MixedRangePolicy::Last,
1616 Self::Changes
1617 | Self::Resets
1618 | Self::CountOverTime
1619 | Self::AbsentOverTime
1620 | Self::PresentOverTime => MixedRangePolicy::Combined,
1621 Self::MinOverTime
1622 | Self::MaxOverTime
1623 | Self::StddevOverTime
1624 | Self::StdvarOverTime
1625 | Self::QuantileOverTime
1626 | Self::Deriv
1627 | Self::PredictLinear
1628 | Self::DoubleExponentialSmoothing
1629 | Self::HoltWinters => MixedRangePolicy::FloatOnly,
1630 }
1631 }
1632
1633 fn float_udf(self) -> Option<ScalarUDF> {
1634 match self {
1635 Self::Rate => Some(Rate::scalar_udf()),
1636 Self::Increase => Some(Increase::scalar_udf()),
1637 Self::Delta => Some(crate::functions::Delta::scalar_udf()),
1638 Self::IDelta => Some(IDelta::<false>::scalar_udf()),
1639 Self::IRate => Some(IDelta::<true>::scalar_udf()),
1640 Self::AvgOverTime => Some(AvgOverTime::scalar_udf()),
1641 Self::MinOverTime => Some(MinOverTime::scalar_udf()),
1642 Self::MaxOverTime => Some(MaxOverTime::scalar_udf()),
1643 Self::SumOverTime => Some(SumOverTime::scalar_udf()),
1644 Self::LastOverTime => Some(LastOverTime::scalar_udf()),
1645 Self::StddevOverTime => Some(StddevOverTime::scalar_udf()),
1646 Self::StdvarOverTime => Some(StdvarOverTime::scalar_udf()),
1647 Self::QuantileOverTime => Some(QuantileOverTime::scalar_udf()),
1648 Self::Deriv => Some(Deriv::scalar_udf()),
1649 Self::PredictLinear => Some(PredictLinear::scalar_udf()),
1650 Self::DoubleExponentialSmoothing | Self::HoltWinters => {
1651 Some(DoubleExponentialSmoothing::scalar_udf())
1652 }
1653 Self::Changes
1654 | Self::Resets
1655 | Self::CountOverTime
1656 | Self::AbsentOverTime
1657 | Self::PresentOverTime => None,
1658 }
1659 }
1660
1661 fn histogram_udf(self, collector: Option<PromqlAnnotationCollector>) -> Option<ScalarUDF> {
1662 match self {
1663 Self::Rate => Some(NativeHistogramRate::scalar_udf_with_collector(collector)),
1664 Self::Increase => Some(NativeHistogramIncrease::scalar_udf_with_collector(
1665 collector,
1666 )),
1667 Self::Delta => Some(NativeHistogramDelta::scalar_udf_with_collector(collector)),
1668 Self::IDelta => Some(NativeHistogramIDelta::scalar_udf_with_collector(collector)),
1669 Self::IRate => Some(NativeHistogramIRate::scalar_udf_with_collector(collector)),
1670 Self::AvgOverTime => Some(NativeHistogramAvgOverTime::scalar_udf_with_collector(
1671 collector,
1672 )),
1673 Self::SumOverTime => Some(NativeHistogramSumOverTime::scalar_udf_with_collector(
1674 collector,
1675 )),
1676 Self::LastOverTime => Some(NativeHistogramLastOverTime::scalar_udf()),
1677 _ => None,
1678 }
1679 }
1680}
1681
1682#[derive(Debug, Clone, Copy)]
1683enum MixedRangePolicy {
1684 DropMixed,
1685 LastTwo,
1686 Last,
1687 Combined,
1688 FloatOnly,
1689}
1690
1691#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1692enum SampleLane {
1693 Float,
1694 Histogram,
1695}
1696
1697fn mixed_range(
1698 args: &ScalarFunctionArgs,
1699 output: MixedRangeOutput,
1700 collector: Option<PromqlAnnotationCollector>,
1701) -> DfResult<ColumnarValue> {
1702 if args.args.len() < 4 {
1703 return Err(DataFusionError::Plan(format!(
1704 "{} function should have at least 4 inputs",
1705 output.name()
1706 )));
1707 }
1708 let function = MixedRangeFunction::parse(&args.args[0])?;
1709 let name = function.name();
1710 let ts_range = extract_range_dict(
1711 &args.args[1],
1712 name,
1713 "timestamp range vector",
1714 &DataType::Timestamp(TimeUnit::Millisecond, None),
1715 )?;
1716 let float_range = extract_range_dict(
1717 &args.args[2],
1718 name,
1719 "float range vector",
1720 &DataType::Float64,
1721 )?;
1722 let histogram_range = extract_range_dict(
1723 &args.args[3],
1724 name,
1725 "native histogram range vector",
1726 &native_histogram_arrow_type(),
1727 )?;
1728 let keys = ts_range.keys().values();
1729 if float_range.keys().values() != keys || histogram_range.keys().values() != keys {
1730 return Err(DataFusionError::Execution(format!(
1731 "{name}: timestamp, float, and native histogram ranges should have the same layout"
1732 )));
1733 }
1734 if args.number_rows != keys.len() {
1735 return Err(DataFusionError::Execution(format!(
1736 "{name}: range inputs have {} windows but the batch has {} rows",
1737 keys.len(),
1738 args.number_rows
1739 )));
1740 }
1741
1742 let timestamps = ts_range
1743 .values()
1744 .as_any()
1745 .downcast_ref::<TimestampMillisecondArray>()
1746 .expect("validated timestamp range");
1747 let floats = float_range
1748 .values()
1749 .as_any()
1750 .downcast_ref::<Float64Array>()
1751 .expect("validated float range");
1752 let histograms = histogram_range
1753 .values()
1754 .as_any()
1755 .downcast_ref::<StructArray>()
1756 .expect("validated native histogram range");
1757 if floats.len() != timestamps.len() || histograms.len() != timestamps.len() {
1758 return Err(DataFusionError::Execution(format!(
1759 "{name}: timestamp, float, and native histogram values should be row-aligned"
1760 )));
1761 }
1762
1763 let bounds = checked_window_bounds(keys, timestamps.len(), name)?;
1764 let float_valid = (0..floats.len())
1765 .map(|row| floats.is_valid(row))
1766 .collect::<Vec<_>>();
1767 let histogram_valid = (0..histograms.len())
1768 .map(|row| histograms.is_valid(row))
1769 .collect::<Vec<_>>();
1770 let float_prefix = validity_prefix(&float_valid, name)?;
1771 let histogram_prefix = validity_prefix(&histogram_valid, name)?;
1772
1773 if matches!(function.policy(), MixedRangePolicy::Combined) {
1774 if output != MixedRangeOutput::Float {
1775 return Err(DataFusionError::Execution(format!(
1776 "{name} does not return native histograms"
1777 )));
1778 }
1779 return combined_range_float(
1780 function,
1781 &bounds,
1782 timestamps,
1783 floats,
1784 histograms,
1785 &float_valid,
1786 &histogram_valid,
1787 &float_prefix,
1788 &histogram_prefix,
1789 );
1790 }
1791
1792 let selections = select_mixed_range_lanes(
1793 function,
1794 &bounds,
1795 timestamps,
1796 &float_valid,
1797 &histogram_valid,
1798 &float_prefix,
1799 &histogram_prefix,
1800 &collector,
1801 );
1802 let (lane, values, valid, prefix) = match output {
1803 MixedRangeOutput::Float => (
1804 SampleLane::Float,
1805 floats as &dyn Array,
1806 float_valid.as_slice(),
1807 float_prefix.as_slice(),
1808 ),
1809 MixedRangeOutput::Histogram => (
1810 SampleLane::Histogram,
1811 histograms as &dyn Array,
1812 histogram_valid.as_slice(),
1813 histogram_prefix.as_slice(),
1814 ),
1815 };
1816 let input = compact_lane_input(
1817 timestamps,
1818 values,
1819 valid,
1820 prefix,
1821 &bounds,
1822 &selections,
1823 lane,
1824 name,
1825 )?;
1826
1827 let udf = match output {
1828 MixedRangeOutput::Float => function.float_udf(),
1829 MixedRangeOutput::Histogram => function.histogram_udf(collector),
1830 }
1831 .ok_or_else(|| {
1832 DataFusionError::Execution(format!(
1833 "{name} does not return {} values",
1834 match output {
1835 MixedRangeOutput::Float => "float",
1836 MixedRangeOutput::Histogram => "native histogram",
1837 }
1838 ))
1839 })?;
1840 let mut input = input;
1841 input.extend_from_slice(&args.args[4..]);
1842 invoke_range_udf(udf, input, args, bounds.len())
1843}
1844
1845fn checked_window_bounds(
1847 keys: &[i64],
1848 value_len: usize,
1849 name: &str,
1850) -> DfResult<Vec<(usize, usize)>> {
1851 keys.iter()
1852 .map(|key| {
1853 let (offset, length) = unpack(*key);
1854 let offset = offset as usize;
1855 let end = offset
1856 .checked_add(length as usize)
1857 .filter(|end| *end <= value_len)
1858 .ok_or_else(|| {
1859 DataFusionError::Execution(format!(
1860 "{name}: invalid range ({offset}, {length}) for {value_len} values"
1861 ))
1862 })?;
1863 Ok((offset, end))
1864 })
1865 .collect()
1866}
1867
1868fn validity_prefix(valid: &[bool], name: &str) -> DfResult<Vec<usize>> {
1870 let capacity = valid.len().checked_add(1).ok_or_else(|| {
1871 DataFusionError::Execution(format!("{name}: sample validity length overflow"))
1872 })?;
1873 let mut prefix = Vec::with_capacity(capacity);
1874 prefix.push(0usize);
1875 for is_valid in valid {
1876 prefix.push(
1877 prefix
1878 .last()
1879 .copied()
1880 .unwrap()
1881 .checked_add(usize::from(*is_valid))
1882 .ok_or_else(|| {
1883 DataFusionError::Execution(format!("{name}: sample count overflow"))
1884 })?,
1885 );
1886 }
1887 Ok(prefix)
1888}
1889
1890fn valid_count(prefix: &[usize], offset: usize, end: usize) -> usize {
1892 prefix[end]
1893 .checked_sub(prefix[offset])
1894 .expect("validity prefix is monotonic")
1895}
1896
1897#[allow(clippy::too_many_arguments)]
1899fn select_mixed_range_lanes(
1900 function: MixedRangeFunction,
1901 bounds: &[(usize, usize)],
1902 timestamps: &TimestampMillisecondArray,
1903 float_valid: &[bool],
1904 histogram_valid: &[bool],
1905 float_prefix: &[usize],
1906 histogram_prefix: &[usize],
1907 collector: &Option<PromqlAnnotationCollector>,
1908) -> Vec<Option<SampleLane>> {
1909 bounds
1910 .iter()
1911 .map(|(offset, end)| {
1912 let float_count = valid_count(float_prefix, *offset, *end);
1913 let histogram_count = valid_count(histogram_prefix, *offset, *end);
1914 match function.policy() {
1915 MixedRangePolicy::DropMixed => match (float_count > 0, histogram_count > 0) {
1916 (true, true) => {
1917 record_warning(
1918 collector,
1919 format!(
1920 "{}: encountered a mix of float and native histogram samples",
1921 function.name()
1922 ),
1923 );
1924 None
1925 }
1926 (true, false) => Some(SampleLane::Float),
1927 (false, true) => Some(SampleLane::Histogram),
1928 (false, false) => None,
1929 },
1930 MixedRangePolicy::FloatOnly => {
1931 if float_count > 0 && histogram_count > 0 {
1932 record_info(
1933 collector,
1934 format!(
1935 "{}: ignored native histogram samples",
1936 function.name()
1937 ),
1938 );
1939 }
1940 (float_count > 0).then_some(SampleLane::Float)
1941 }
1942 MixedRangePolicy::Last => (*offset..*end).rev().find_map(|row| {
1943 if histogram_valid[row] {
1944 Some(SampleLane::Histogram)
1945 } else if float_valid[row] {
1946 Some(SampleLane::Float)
1947 } else {
1948 None
1949 }
1950 }),
1951 MixedRangePolicy::LastTwo => {
1952 let mut last_two = Vec::with_capacity(2);
1953 for row in (*offset..*end).rev() {
1954 if float_valid[row] {
1957 last_two.push((SampleLane::Float, timestamps.value(row)));
1958 }
1959 if last_two.len() < 2 && histogram_valid[row] {
1960 last_two.push((SampleLane::Histogram, timestamps.value(row)));
1961 }
1962 if last_two.len() == 2 {
1963 break;
1964 }
1965 }
1966 if last_two.len() < 2 || last_two[0].1 == last_two[1].1 {
1967 None
1968 } else if last_two[0].0 == last_two[1].0 {
1969 Some(last_two[0].0)
1970 } else {
1971 record_warning(
1972 collector,
1973 format!(
1974 "{}: encountered a mix of float and native histogram samples in the last two points",
1975 function.name()
1976 ),
1977 );
1978 None
1979 }
1980 }
1981 MixedRangePolicy::Combined => unreachable!(),
1982 }
1983 })
1984 .collect()
1985}
1986
1987#[allow(clippy::too_many_arguments)]
1990fn compact_lane_input(
1991 timestamps: &TimestampMillisecondArray,
1992 values: &dyn Array,
1993 valid: &[bool],
1994 prefix: &[usize],
1995 bounds: &[(usize, usize)],
1996 selections: &[Option<SampleLane>],
1997 lane: SampleLane,
1998 name: &str,
1999) -> DfResult<Vec<ColumnarValue>> {
2000 let mask = BooleanArray::from(valid.to_vec());
2001 let filtered_timestamps = filter(timestamps, &mask)?;
2002 let filtered_values = filter(values, &mask)?;
2003 let ranges = bounds
2004 .iter()
2005 .zip(selections)
2006 .map(|((offset, end), selected)| {
2007 let compact_offset = prefix[*offset];
2008 let compact_length = if *selected == Some(lane) {
2009 valid_count(prefix, *offset, *end)
2010 } else {
2011 0
2012 };
2013 Ok((
2014 u32::try_from(compact_offset).map_err(|_| {
2015 DataFusionError::Execution(format!(
2016 "{name}: compacted range offset exceeds u32"
2017 ))
2018 })?,
2019 u32::try_from(compact_length).map_err(|_| {
2020 DataFusionError::Execution(format!(
2021 "{name}: compacted range length exceeds u32"
2022 ))
2023 })?,
2024 ))
2025 })
2026 .collect::<DfResult<Vec<_>>>()?;
2027 let timestamp_range = RangeArray::from_ranges(filtered_timestamps, ranges.clone())
2028 .map_err(DataFusionError::from)?;
2029 let value_range =
2030 RangeArray::from_ranges(filtered_values, ranges).map_err(DataFusionError::from)?;
2031 Ok(vec![
2032 ColumnarValue::Array(Arc::new(timestamp_range.into_dict())),
2033 ColumnarValue::Array(Arc::new(value_range.into_dict())),
2034 ])
2035}
2036
2037fn invoke_range_udf(
2038 udf: ScalarUDF,
2039 input: Vec<ColumnarValue>,
2040 outer_args: &ScalarFunctionArgs,
2041 number_rows: usize,
2042) -> DfResult<ColumnarValue> {
2043 let arg_fields = input
2044 .iter()
2045 .enumerate()
2046 .map(|(index, value)| Arc::new(Field::new(format!("arg_{index}"), value.data_type(), true)))
2047 .collect();
2048 udf.invoke_with_args(ScalarFunctionArgs {
2049 args: input,
2050 arg_fields,
2051 number_rows,
2052 return_field: outer_args.return_field.clone(),
2053 config_options: outer_args.config_options.clone(),
2054 })
2055}
2056
2057enum MixedSample {
2058 Float(f64),
2059 Histogram(NativeHistogram),
2060}
2061
2062#[allow(clippy::too_many_arguments)]
2063fn combined_range_float(
2064 function: MixedRangeFunction,
2065 bounds: &[(usize, usize)],
2066 timestamps: &TimestampMillisecondArray,
2067 floats: &Float64Array,
2068 histograms: &StructArray,
2069 float_valid: &[bool],
2070 histogram_valid: &[bool],
2071 float_prefix: &[usize],
2072 histogram_prefix: &[usize],
2073) -> DfResult<ColumnarValue> {
2074 let mut result = Float64Builder::with_capacity(bounds.len());
2075 for (offset, end) in bounds {
2076 let sample_count = valid_count(float_prefix, *offset, *end)
2077 .checked_add(valid_count(histogram_prefix, *offset, *end))
2078 .ok_or_else(|| {
2079 DataFusionError::Execution(format!("{}: sample count overflow", function.name()))
2080 })?;
2081 match function {
2082 MixedRangeFunction::CountOverTime => {
2083 if sample_count == 0 {
2084 result.append_null();
2085 } else {
2086 result.append_value(sample_count as f64);
2087 }
2088 }
2089 MixedRangeFunction::AbsentOverTime => {
2090 if sample_count == 0 {
2091 result.append_value(1.0);
2092 } else {
2093 result.append_null();
2094 }
2095 }
2096 MixedRangeFunction::PresentOverTime => {
2097 if sample_count == 0 {
2098 result.append_null();
2099 } else {
2100 result.append_value(1.0);
2101 }
2102 }
2103 MixedRangeFunction::Changes | MixedRangeFunction::Resets => {
2104 if sample_count == 0 {
2105 result.append_null();
2106 continue;
2107 }
2108 let mut count = 0usize;
2109 let mut previous = None;
2110 for row in *offset..*end {
2111 if float_valid[row] {
2112 count += mixed_transition(
2113 function,
2114 &mut previous,
2115 timestamps.value(row),
2116 MixedSample::Float(floats.value(row)),
2117 );
2118 }
2119 if histogram_valid[row] {
2120 let histogram = read_histogram(histograms, row)?
2121 .expect("validated native histogram sample");
2122 count += mixed_transition(
2123 function,
2124 &mut previous,
2125 timestamps.value(row),
2126 MixedSample::Histogram(histogram),
2127 );
2128 }
2129 }
2130 result.append_value(count as f64);
2131 }
2132 _ => {
2133 return Err(DataFusionError::Internal(format!(
2134 "{} does not support combined range evaluation",
2135 function.name()
2136 )));
2137 }
2138 }
2139 }
2140 Ok(ColumnarValue::Array(Arc::new(result.finish())))
2141}
2142
2143fn mixed_transition(
2144 function: MixedRangeFunction,
2145 previous: &mut Option<(i64, MixedSample)>,
2146 timestamp: i64,
2147 current: MixedSample,
2148) -> usize {
2149 let changed = previous.as_ref().is_some_and(|(previous_ts, previous)| {
2150 match (function, previous, ¤t) {
2151 (
2152 MixedRangeFunction::Changes,
2153 MixedSample::Float(previous),
2154 MixedSample::Float(current),
2155 ) => current != previous && !(current.is_nan() && previous.is_nan()),
2156 (
2157 MixedRangeFunction::Changes,
2158 MixedSample::Histogram(previous),
2159 MixedSample::Histogram(current),
2160 ) => !current.promql_eq(previous),
2161 (MixedRangeFunction::Changes, _, _) => true,
2162 (
2163 MixedRangeFunction::Resets,
2164 MixedSample::Float(previous),
2165 MixedSample::Float(current),
2166 ) => current < previous,
2167 (
2168 MixedRangeFunction::Resets,
2169 MixedSample::Histogram(previous),
2170 MixedSample::Histogram(current),
2171 ) => {
2172 (previous.reset_hint == GAUGE_RESET_HINT)
2173 != (current.reset_hint == GAUGE_RESET_HINT)
2174 || current.detect_counter_reset(previous, *previous_ts, timestamp)
2175 }
2176 (MixedRangeFunction::Resets, _, _) => true,
2177 _ => unreachable!(),
2178 }
2179 });
2180 *previous = Some((timestamp, current));
2181 usize::from(changed)
2182}
2183
2184fn native_histogram_scalar(histogram: Option<NativeHistogram>) -> ScalarValue {
2185 let array = build_histogram_array(&[histogram]);
2186 let histogram = array
2187 .as_any()
2188 .downcast_ref::<StructArray>()
2189 .expect("native histogram array is a StructArray")
2190 .clone();
2191 ScalarValue::Struct(Arc::new(histogram))
2192}
2193
2194fn native_histogram_aggregate_udf(
2195 kind: NativeHistogramAggregateKind,
2196 collector: Option<PromqlAnnotationCollector>,
2197) -> AggregateUDF {
2198 let state_types = if kind.needs_count() {
2199 vec![
2200 native_histogram_arrow_type(),
2201 DataType::UInt64,
2202 DataType::Boolean,
2203 DataType::Boolean,
2204 DataType::Boolean,
2205 ]
2206 } else {
2207 vec![
2208 native_histogram_arrow_type(),
2209 DataType::Boolean,
2210 DataType::Boolean,
2211 DataType::Boolean,
2212 ]
2213 };
2214
2215 create_udaf(
2216 kind.name(),
2217 vec![native_histogram_arrow_type()],
2218 Arc::new(native_histogram_arrow_type()),
2219 Volatility::Volatile,
2220 Arc::new(move |args| {
2221 NativeHistogramAggregateAccumulator::from_args(kind, collector.clone(), args)
2222 }),
2223 Arc::new(state_types),
2224 )
2225}
2226
2227impl DfAccumulator for NativeHistogramAggregateAccumulator {
2228 fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
2229 let histograms = values
2230 .first()
2231 .and_then(|array| array.as_any().downcast_ref::<StructArray>())
2232 .ok_or_else(|| {
2233 DataFusionError::Execution(format!(
2234 "{}: expected native histogram struct input",
2235 self.kind.name()
2236 ))
2237 })?;
2238
2239 for row in 0..histograms.len() {
2240 let Some(histogram) = read_histogram(histograms, row)? else {
2241 continue;
2242 };
2243 self.push_histogram(histogram, 1)?;
2244 }
2245
2246 Ok(())
2247 }
2248
2249 fn evaluate(&mut self) -> DfResult<ScalarValue> {
2250 let histogram = match (self.kind, self.dropped_incompatible, self.value.clone()) {
2251 (_, true, _) => None,
2252 (_, false, None) => None,
2253 (NativeHistogramAggregateKind::Sum, false, value) => value,
2254 (NativeHistogramAggregateKind::Avg, false, Some(value)) if self.count > 0 => {
2255 Some(value)
2256 }
2257 (NativeHistogramAggregateKind::Avg, _, _) => None,
2258 };
2259
2260 Ok(native_histogram_scalar(histogram))
2261 }
2262
2263 fn size(&self) -> usize {
2264 size_of::<Self>()
2265 + self.value.as_ref().map_or(0, |histogram| {
2266 histogram.custom_values.capacity() * size_of::<f64>()
2267 + histogram.positive_spans.capacity() * size_of::<Span>()
2268 + histogram.negative_spans.capacity() * size_of::<Span>()
2269 + histogram.positive_buckets.capacity() * size_of::<f64>()
2270 + histogram.negative_buckets.capacity() * size_of::<f64>()
2271 })
2272 }
2273
2274 fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
2275 let mut state = vec![native_histogram_scalar(self.value.clone())];
2276 if self.kind.needs_count() {
2277 state.push(ScalarValue::UInt64(Some(self.count)));
2278 }
2279 state.push(ScalarValue::Boolean(Some(self.dropped_incompatible)));
2280 state.push(ScalarValue::Boolean(Some(self.counter_reset_seen)));
2281 state.push(ScalarValue::Boolean(Some(self.not_counter_reset_seen)));
2282 Ok(state)
2283 }
2284
2285 fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> {
2286 if states.is_empty() {
2287 return Ok(());
2288 }
2289
2290 let histograms = states[0]
2291 .as_any()
2292 .downcast_ref::<StructArray>()
2293 .ok_or_else(|| {
2294 DataFusionError::Execution(format!(
2295 "{}: expected native histogram struct state",
2296 self.kind.name()
2297 ))
2298 })?;
2299 let counts = if self.kind.needs_count() {
2300 Some(
2301 states
2302 .get(1)
2303 .and_then(|array| array.as_any().downcast_ref::<UInt64Array>())
2304 .ok_or_else(|| {
2305 DataFusionError::Execution(format!(
2306 "{}: expected UInt64 count state",
2307 self.kind.name()
2308 ))
2309 })?,
2310 )
2311 } else {
2312 None
2313 };
2314 let dropped_index = if self.kind.needs_count() { 2 } else { 1 };
2315 let dropped = states
2316 .get(dropped_index)
2317 .and_then(|array| array.as_any().downcast_ref::<BooleanArray>())
2318 .ok_or_else(|| {
2319 DataFusionError::Execution(format!(
2320 "{}: expected Boolean dropped state",
2321 self.kind.name()
2322 ))
2323 })?;
2324 let counter_reset_seen = states
2325 .get(dropped_index + 1)
2326 .and_then(|array| array.as_any().downcast_ref::<BooleanArray>())
2327 .ok_or_else(|| {
2328 DataFusionError::Execution(format!(
2329 "{}: expected Boolean counter reset state",
2330 self.kind.name()
2331 ))
2332 })?;
2333 let not_counter_reset_seen = states
2334 .get(dropped_index + 2)
2335 .and_then(|array| array.as_any().downcast_ref::<BooleanArray>())
2336 .ok_or_else(|| {
2337 DataFusionError::Execution(format!(
2338 "{}: expected Boolean not-counter-reset state",
2339 self.kind.name()
2340 ))
2341 })?;
2342
2343 for row in 0..histograms.len() {
2344 self.observe_reset_hints(
2345 counter_reset_seen.value(row),
2346 not_counter_reset_seen.value(row),
2347 );
2348 if dropped.value(row) {
2349 self.mark_incompatible();
2350 }
2351 if self.dropped_incompatible {
2352 continue;
2353 }
2354 let Some(histogram) = read_histogram(histograms, row)? else {
2355 continue;
2356 };
2357 let count = counts.map(|counts| counts.value(row)).unwrap_or(1);
2358 self.push_histogram(histogram, count)?;
2359 }
2360
2361 Ok(())
2362 }
2363}
2364
2365fn histogram_delta(
2366 samples: &[NativeHistogram],
2367 timestamps: &[i64],
2368 is_counter: bool,
2369) -> Option<NativeHistogram> {
2370 if samples.len() < 2 || samples.len() != timestamps.len() {
2371 return None;
2372 }
2373
2374 if !is_counter {
2375 return samples
2376 .last()?
2377 .sub(samples.first()?)
2378 .map(NativeHistogram::into_gauge);
2379 }
2380
2381 let first_reset = samples[1].detect_counter_reset(&samples[0], timestamps[0], timestamps[1]);
2382 let (initial, reset_scan_start) = if first_reset {
2383 (samples[1].zero_like(), 2)
2386 } else {
2387 (samples[0].clone(), 1)
2388 };
2389 let mut result = samples.last()?.sub(&initial)?;
2390 for index in reset_scan_start..samples.len() {
2391 if samples[index].detect_counter_reset(
2392 &samples[index - 1],
2393 timestamps[index - 1],
2394 timestamps[index],
2395 ) {
2396 result = result.add(&samples[index - 1])?;
2397 }
2398 }
2399 Some(result.into_gauge())
2400}
2401
2402fn idelta_value(
2403 samples: &[NativeHistogram],
2404 is_rate: bool,
2405 previous_ts: i64,
2406 current_ts: i64,
2407 sampled_interval_secs: f64,
2408) -> Option<NativeHistogram> {
2409 if samples.len() < 2 {
2410 return None;
2411 }
2412 let previous = &samples[samples.len() - 2];
2413 let current = samples.last()?;
2414 let result = if is_rate && current.detect_counter_reset(previous, previous_ts, current_ts) {
2415 current.clone()
2416 } else {
2417 current.sub(previous)?
2418 };
2419 Some(
2420 if is_rate {
2421 result.scale(1.0 / sampled_interval_secs)
2422 } else {
2423 result
2424 }
2425 .into_gauge(),
2426 )
2427}
2428
2429fn native_extrapolated_rate<const IS_COUNTER: bool, const IS_RATE: bool>(
2430 input: &[ColumnarValue],
2431 range_length: i64,
2432 func_name: &'static str,
2433 collector: Option<PromqlAnnotationCollector>,
2434) -> DfResult<ColumnarValue> {
2435 if input.len() != 4 {
2436 return Err(DataFusionError::Plan(format!(
2437 "{func_name} function should have 4 inputs"
2438 )));
2439 }
2440
2441 let ts_dict = extract_range_dict(
2442 &input[0],
2443 func_name,
2444 "timestamp range vector",
2445 &DataType::Timestamp(TimeUnit::Millisecond, None),
2446 )?;
2447 let value_dict = extract_range_dict(
2448 &input[1],
2449 func_name,
2450 "value range vector",
2451 &native_histogram_arrow_type(),
2452 )?;
2453 let eval_ts = extract_array(&input[2])?;
2454 let eval_ts = eval_ts
2455 .as_any()
2456 .downcast_ref::<TimestampMillisecondArray>()
2457 .ok_or_else(|| {
2458 DataFusionError::Execution(format!(
2459 "{func_name}: expect evaluation timestamp vector as Timestamp(Millisecond), found {}",
2460 eval_ts.data_type()
2461 ))
2462 })?;
2463
2464 let keys = ts_dict.keys().values();
2465 if value_dict.keys().values() != keys || eval_ts.len() != keys.len() {
2466 return Err(DataFusionError::Execution(format!(
2467 "{func_name}: timestamp, value, and evaluation ranges should have the same layout"
2468 )));
2469 }
2470
2471 let all_timestamps = ts_dict
2472 .values()
2473 .as_any()
2474 .downcast_ref::<TimestampMillisecondArray>()
2475 .expect("validated timestamp range")
2476 .values();
2477 let all_histograms = value_dict
2478 .values()
2479 .as_any()
2480 .downcast_ref::<StructArray>()
2481 .expect("validated native histogram range");
2482 let range_length_secs = range_length as f64 / 1000.0;
2483 let mut result = Vec::with_capacity(keys.len());
2484
2485 for index in 0..keys.len() {
2486 let (raw_offset, raw_length) = unpack(keys[index]);
2487 let offset = raw_offset as usize;
2488 let length = raw_length as usize;
2489 if length == 0 {
2490 result.push(None);
2491 continue;
2492 }
2493
2494 let mut samples = Vec::with_capacity(length);
2495 let mut has_null = false;
2496 for row in offset..offset + length {
2497 let Some(histogram) = read_histogram(all_histograms, row)? else {
2498 has_null = true;
2499 break;
2500 };
2501 samples.push(histogram);
2502 }
2503 if has_null {
2504 result.push(None);
2505 continue;
2506 }
2507
2508 let first_ts = all_timestamps[offset];
2509 let last_ts = all_timestamps[offset + length - 1];
2510 let range_end = eval_ts.value(index);
2511 let range_start = range_end - range_length;
2512 let synthetic_zero_timestamp = IS_COUNTER
2513 .then_some(samples[0].start_timestamp)
2514 .flatten()
2515 .filter(|start| *start != 0 && range_start < *start && *start < first_ts);
2516 let synthetic_zero_start = synthetic_zero_timestamp.is_some();
2517 if length < 2 && !synthetic_zero_start {
2518 result.push(None);
2519 continue;
2520 }
2521
2522 let wrong_flavor = if IS_COUNTER {
2523 samples
2524 .iter()
2525 .any(|histogram| histogram.reset_hint == GAUGE_RESET_HINT)
2526 } else {
2527 samples[0].reset_hint != GAUGE_RESET_HINT
2528 || samples[samples.len() - 1].reset_hint != GAUGE_RESET_HINT
2529 };
2530 if wrong_flavor {
2531 let expected = if IS_COUNTER { "counter" } else { "gauge" };
2532 record_warning(
2533 &collector,
2534 format!("{func_name}: native histogram input should be a {expected} histogram"),
2535 );
2536 }
2537
2538 let timestamps = &all_timestamps[offset..offset + length];
2539 for pair in samples.windows(2) {
2540 record_custom_reconciliation(&collector, func_name, &pair[0], &pair[1]);
2541 }
2542 let mut histogram = if length == 1 {
2543 samples[0].clone()
2544 } else if let Some(histogram) = histogram_delta(&samples, timestamps, IS_COUNTER) {
2545 histogram
2546 } else {
2547 record_warning(
2548 &collector,
2549 format!("{func_name}: dropped native histogram range with incompatible schemas"),
2550 );
2551 result.push(None);
2552 continue;
2553 };
2554 if synthetic_zero_start && length > 1 {
2555 let Some(with_synthetic_zero) = histogram.add(&samples[0]) else {
2556 record_warning(
2557 &collector,
2558 format!(
2559 "{func_name}: dropped native histogram range with incompatible schemas"
2560 ),
2561 );
2562 result.push(None);
2563 continue;
2564 };
2565 histogram = with_synthetic_zero;
2566 }
2567
2568 let real_sampled_interval_ms = (last_ts - first_ts) as f64;
2569 let sampled_interval_ms = synthetic_zero_timestamp
2570 .map(|start| (last_ts - start) as f64)
2571 .unwrap_or(real_sampled_interval_ms);
2572 if sampled_interval_ms <= 0.0 {
2573 result.push(None);
2574 continue;
2575 }
2576 let average_interval_ms = if length > 1 {
2577 real_sampled_interval_ms / (length - 1) as f64
2578 } else {
2579 0.0
2580 };
2581 let mut duration_to_start_ms = if synthetic_zero_start {
2582 0.0
2583 } else {
2584 (first_ts - range_start) as f64
2585 };
2586 let duration_to_end_ms = (range_end - last_ts) as f64;
2587
2588 if IS_COUNTER && !synthetic_zero_start && histogram.count > 0.0 && samples[0].count >= 0.0 {
2589 let duration_to_zero = sampled_interval_ms * (samples[0].count / histogram.count);
2590 if duration_to_zero < duration_to_start_ms {
2591 duration_to_start_ms = duration_to_zero;
2592 }
2593 }
2594
2595 let extrapolation_threshold = average_interval_ms * 1.1;
2596 let mut extrapolated_interval_ms = sampled_interval_ms;
2597 if duration_to_start_ms < extrapolation_threshold {
2598 extrapolated_interval_ms += duration_to_start_ms;
2599 } else {
2600 extrapolated_interval_ms += average_interval_ms / 2.0;
2601 }
2602 if duration_to_end_ms < extrapolation_threshold {
2603 extrapolated_interval_ms += duration_to_end_ms;
2604 } else {
2605 extrapolated_interval_ms += average_interval_ms / 2.0;
2606 }
2607
2608 let mut factor = extrapolated_interval_ms / sampled_interval_ms;
2609 if IS_RATE {
2610 factor /= range_length_secs;
2611 }
2612 histogram = histogram.scale(factor).into_gauge();
2613 result.push(Some(histogram));
2614 }
2615
2616 Ok(ColumnarValue::Array(build_histogram_array(&result)))
2617}
2618
2619fn create_native_extrapolated_udf<const IS_COUNTER: bool, const IS_RATE: bool>(
2620 name: &'static str,
2621 collector: Option<PromqlAnnotationCollector>,
2622) -> ScalarUDF {
2623 let input_types = vec![
2624 RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
2625 RangeArray::convert_data_type(native_histogram_arrow_type()),
2626 DataType::Timestamp(TimeUnit::Millisecond, None),
2627 DataType::Int64,
2628 ];
2629 create_udf(
2630 name,
2631 input_types,
2632 native_histogram_arrow_type(),
2633 Volatility::Volatile,
2634 Arc::new(move |input: &[ColumnarValue]| {
2635 let range_length = extract_array(&input[3])?;
2636 let range_length = range_length
2637 .as_any()
2638 .downcast_ref::<Int64Array>()
2639 .ok_or_else(|| {
2640 DataFusionError::Execution(format!(
2641 "{name}: expect Int64 as range length type, found {}",
2642 range_length.data_type()
2643 ))
2644 })?;
2645 if range_length.is_empty() || range_length.is_null(0) {
2646 return Err(DataFusionError::Execution(format!(
2647 "{name}: range length must contain a non-null Int64 value"
2648 )));
2649 }
2650 native_extrapolated_rate::<IS_COUNTER, IS_RATE>(
2651 input,
2652 range_length.value(0),
2653 name,
2654 collector.clone(),
2655 )
2656 }) as _,
2657 )
2658}
2659
2660pub struct NativeHistogramDelta;
2661pub struct NativeHistogramRate;
2662pub struct NativeHistogramIncrease;
2663
2664impl NativeHistogramDelta {
2665 pub const fn name() -> &'static str {
2666 "prom_native_histogram_delta"
2667 }
2668
2669 pub fn scalar_udf() -> ScalarUDF {
2670 Self::scalar_udf_with_collector(None)
2671 }
2672
2673 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2674 create_native_extrapolated_udf::<false, false>(Self::name(), collector)
2675 }
2676}
2677
2678impl NativeHistogramRate {
2679 pub const fn name() -> &'static str {
2680 "prom_native_histogram_rate"
2681 }
2682
2683 pub fn scalar_udf() -> ScalarUDF {
2684 Self::scalar_udf_with_collector(None)
2685 }
2686
2687 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2688 create_native_extrapolated_udf::<true, true>(Self::name(), collector)
2689 }
2690}
2691
2692impl NativeHistogramIncrease {
2693 pub const fn name() -> &'static str {
2694 "prom_native_histogram_increase"
2695 }
2696
2697 pub fn scalar_udf() -> ScalarUDF {
2698 Self::scalar_udf_with_collector(None)
2699 }
2700
2701 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2702 create_native_extrapolated_udf::<true, false>(Self::name(), collector)
2703 }
2704}
2705
2706fn native_idelta<const IS_RATE: bool>(
2707 input: &[ColumnarValue],
2708 func_name: &'static str,
2709 collector: Option<PromqlAnnotationCollector>,
2710) -> DfResult<ColumnarValue> {
2711 if input.len() != 2 {
2712 return Err(DataFusionError::Plan(format!(
2713 "{func_name} function should have 2 inputs"
2714 )));
2715 }
2716
2717 let ts_range = extract_range_dict(
2718 &input[0],
2719 func_name,
2720 "timestamp range vector",
2721 &DataType::Timestamp(TimeUnit::Millisecond, None),
2722 )?;
2723 let value_range = extract_range_dict(
2724 &input[1],
2725 func_name,
2726 "value range vector",
2727 &native_histogram_arrow_type(),
2728 )?;
2729
2730 if ts_range.keys().values() != value_range.keys().values() {
2731 return Err(DataFusionError::Execution(format!(
2732 "{func_name}: timestamp and value ranges should have the same window layout"
2733 )));
2734 }
2735
2736 let ts_values = ts_range
2737 .values()
2738 .as_any()
2739 .downcast_ref::<TimestampMillisecondArray>()
2740 .expect("validated timestamp range")
2741 .values();
2742 let histograms = value_range
2743 .values()
2744 .as_any()
2745 .downcast_ref::<StructArray>()
2746 .expect("validated native histogram range");
2747 let mut result = Vec::with_capacity(ts_range.keys().len());
2748
2749 for key in ts_range.keys().values() {
2750 let (offset, length) = unpack(*key);
2751 let offset = offset as usize;
2752 let length = length as usize;
2753 if length < 2 {
2754 result.push(None);
2755 continue;
2756 }
2757
2758 let mut samples = Vec::with_capacity(2);
2759 let mut has_null = false;
2760 for row in offset + length - 2..offset + length {
2761 let Some(histogram) = read_histogram(histograms, row)? else {
2762 has_null = true;
2763 break;
2764 };
2765 samples.push(histogram);
2766 }
2767 if has_null {
2768 result.push(None);
2769 continue;
2770 }
2771
2772 let wrong_flavor = samples.iter().any(|histogram| {
2773 if IS_RATE {
2774 histogram.reset_hint == GAUGE_RESET_HINT
2775 } else {
2776 histogram.reset_hint != GAUGE_RESET_HINT
2777 }
2778 });
2779 if wrong_flavor {
2780 let expected = if IS_RATE { "counter" } else { "gauge" };
2781 record_warning(
2782 &collector,
2783 format!("{func_name}: native histogram input should be a {expected} histogram"),
2784 );
2785 }
2786
2787 let sampled_interval_secs =
2788 (ts_values[offset + length - 1] - ts_values[offset + length - 2]) as f64 / 1000.0;
2789 if sampled_interval_secs <= 0.0 {
2790 result.push(None);
2791 continue;
2792 }
2793 record_custom_reconciliation(&collector, func_name, &samples[0], &samples[1]);
2794 let value = idelta_value(
2795 &samples,
2796 IS_RATE,
2797 ts_values[offset + length - 2],
2798 ts_values[offset + length - 1],
2799 sampled_interval_secs,
2800 );
2801 if value.is_none() {
2802 record_warning(
2803 &collector,
2804 format!("{func_name}: dropped native histogram range with incompatible schemas"),
2805 );
2806 }
2807 result.push(value);
2808 }
2809
2810 Ok(ColumnarValue::Array(build_histogram_array(&result)))
2811}
2812
2813fn create_native_idelta_udf<const IS_RATE: bool>(
2814 name: &'static str,
2815 collector: Option<PromqlAnnotationCollector>,
2816) -> ScalarUDF {
2817 create_udf(
2818 name,
2819 vec![
2820 RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
2821 RangeArray::convert_data_type(native_histogram_arrow_type()),
2822 ],
2823 native_histogram_arrow_type(),
2824 Volatility::Volatile,
2825 Arc::new(move |input: &[ColumnarValue]| {
2826 native_idelta::<IS_RATE>(input, name, collector.clone())
2827 }) as _,
2828 )
2829}
2830
2831pub struct NativeHistogramIDelta;
2832pub struct NativeHistogramIRate;
2833
2834impl NativeHistogramIDelta {
2835 pub const fn name() -> &'static str {
2836 "prom_native_histogram_idelta"
2837 }
2838
2839 pub fn scalar_udf() -> ScalarUDF {
2840 Self::scalar_udf_with_collector(None)
2841 }
2842
2843 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2844 create_native_idelta_udf::<false>(Self::name(), collector)
2845 }
2846}
2847
2848impl NativeHistogramIRate {
2849 pub const fn name() -> &'static str {
2850 "prom_native_histogram_irate"
2851 }
2852
2853 pub fn scalar_udf() -> ScalarUDF {
2854 Self::scalar_udf_with_collector(None)
2855 }
2856
2857 pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2858 create_native_idelta_udf::<true>(Self::name(), collector)
2859 }
2860}
2861
2862#[cfg(test)]
2863mod tests {
2864 use datafusion::arrow::datatypes::Field;
2865 use datafusion_common::config::ConfigOptions;
2866 use datafusion_expr::ScalarFunctionArgs;
2867
2868 use super::*;
2869
2870 fn sample_histogram(count: f64, sum: f64, positive_buckets: Vec<f64>) -> NativeHistogram {
2871 NativeHistogram {
2872 schema: 0,
2873 zero_threshold: 0.0,
2874 sum,
2875 reset_hint: UNKNOWN_COUNTER_RESET_HINT,
2876 start_timestamp: None,
2877 custom_values: Vec::new(),
2878 positive_spans: vec![Span {
2879 offset: 0,
2880 length: positive_buckets.len() as i32,
2881 }],
2882 negative_spans: Vec::new(),
2883 count,
2884 zero_count: 0.0,
2885 positive_buckets,
2886 negative_buckets: Vec::new(),
2887 }
2888 }
2889
2890 fn run_scalar_udf(udf: ScalarUDF, input: Vec<ColumnarValue>) -> f64 {
2891 let result = run_udf(udf, input, DataType::Float64);
2892 extract_array(&result)
2893 .unwrap()
2894 .as_any()
2895 .downcast_ref::<Float64Array>()
2896 .unwrap()
2897 .value(0)
2898 }
2899
2900 fn run_udf(udf: ScalarUDF, input: Vec<ColumnarValue>, return_type: DataType) -> ColumnarValue {
2901 let arg_fields = input
2902 .iter()
2903 .enumerate()
2904 .map(|(idx, input)| Arc::new(Field::new(format!("arg_{idx}"), input.data_type(), true)))
2905 .collect();
2906 let args = ScalarFunctionArgs {
2907 args: input,
2908 arg_fields,
2909 number_rows: 1,
2910 return_field: Arc::new(Field::new("result", return_type, true)),
2911 config_options: Arc::new(ConfigOptions::default()),
2912 };
2913
2914 udf.invoke_with_args(args).unwrap()
2915 }
2916
2917 fn run_histogram_udf(udf: ScalarUDF, input: Vec<ColumnarValue>) -> NativeHistogram {
2918 let result = run_udf(udf, input, native_histogram_arrow_type());
2919 let array = extract_array(&result)
2920 .unwrap()
2921 .as_any()
2922 .downcast_ref::<StructArray>()
2923 .unwrap()
2924 .clone();
2925 read_histogram(&array, 0).unwrap().unwrap()
2926 }
2927
2928 fn evaluated_histogram(
2929 accumulator: &mut NativeHistogramAggregateAccumulator,
2930 ) -> NativeHistogram {
2931 let ScalarValue::Struct(array) = accumulator.evaluate().unwrap() else {
2932 panic!("native histogram accumulator returned a non-struct value");
2933 };
2934 read_histogram(&array, 0).unwrap().unwrap()
2935 }
2936
2937 fn histogram_range_input(values: Vec<Option<NativeHistogram>>) -> Vec<ColumnarValue> {
2938 let timestamps = Arc::new(TimestampMillisecondArray::from_iter(
2939 (0..values.len()).map(|idx| Some((idx as i64 + 1) * 1000)),
2940 ));
2941 let histograms = build_histogram_array(&values);
2942 let range = [(0, values.len() as u32)];
2943 let ts_range = RangeArray::from_ranges(timestamps, range).unwrap();
2944 let value_range = RangeArray::from_ranges(histograms, range).unwrap();
2945
2946 vec![
2947 ColumnarValue::Array(Arc::new(ts_range.into_dict())),
2948 ColumnarValue::Array(Arc::new(value_range.into_dict())),
2949 ]
2950 }
2951
2952 fn mixed_range_input(
2953 name: &str,
2954 floats: Vec<Option<f64>>,
2955 histograms: Vec<Option<NativeHistogram>>,
2956 ) -> Vec<ColumnarValue> {
2957 assert_eq!(floats.len(), histograms.len());
2958 let timestamps = Arc::new(TimestampMillisecondArray::from_iter(
2959 (0..floats.len()).map(|idx| Some((idx as i64 + 1) * 1000)),
2960 ));
2961 let floats = Arc::new(Float64Array::from(floats));
2962 let histograms = build_histogram_array(&histograms);
2963 let range = [(0, u32::try_from(floats.len()).unwrap())];
2964 vec![
2965 ColumnarValue::Scalar(ScalarValue::Utf8(Some(name.to_string()))),
2966 ColumnarValue::Array(Arc::new(
2967 RangeArray::from_ranges(timestamps, range)
2968 .unwrap()
2969 .into_dict(),
2970 )),
2971 ColumnarValue::Array(Arc::new(
2972 RangeArray::from_ranges(floats, range).unwrap().into_dict(),
2973 )),
2974 ColumnarValue::Array(Arc::new(
2975 RangeArray::from_ranges(histograms, range)
2976 .unwrap()
2977 .into_dict(),
2978 )),
2979 ]
2980 }
2981
2982 fn mixed_float_result(udf: ScalarUDF, input: Vec<ColumnarValue>) -> Option<f64> {
2983 let result = run_udf(udf, input, DataType::Float64);
2984 let result = extract_array(&result).unwrap();
2985 let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
2986 result.is_valid(0).then(|| result.value(0))
2987 }
2988
2989 fn mixed_histogram_result(
2990 udf: ScalarUDF,
2991 input: Vec<ColumnarValue>,
2992 ) -> Option<NativeHistogram> {
2993 let result = run_udf(udf, input, native_histogram_arrow_type());
2994 let result = extract_array(&result).unwrap();
2995 let result = result.as_any().downcast_ref::<StructArray>().unwrap();
2996 read_histogram(result, 0).unwrap()
2997 }
2998
2999 fn run_histogram_range_udf(
3000 udf: ScalarUDF,
3001 histograms: Vec<NativeHistogram>,
3002 ) -> NativeHistogram {
3003 run_histogram_udf(
3004 udf,
3005 histogram_range_input(histograms.into_iter().map(Some).collect()),
3006 )
3007 }
3008
3009 fn run_float_range_udf(
3010 udf: ScalarUDF,
3011 histograms: Vec<Option<NativeHistogram>>,
3012 ) -> Option<f64> {
3013 let result = run_udf(udf, histogram_range_input(histograms), DataType::Float64);
3014 let result = extract_array(&result).unwrap();
3015 let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
3016 (!result.is_null(0)).then(|| result.value(0))
3017 }
3018
3019 fn run_extrapolated_histogram_udf(
3020 udf: ScalarUDF,
3021 histograms: Vec<NativeHistogram>,
3022 ) -> NativeHistogram {
3023 let range_length = histograms.len() as i64 * 1000;
3024 let timestamps = (0..histograms.len())
3025 .map(|idx| (idx as i64 + 1) * 1000)
3026 .collect();
3027 extrapolated_histogram_result(udf, timestamps, histograms, range_length, range_length)
3028 .unwrap()
3029 }
3030
3031 fn extrapolated_histogram_result(
3032 udf: ScalarUDF,
3033 timestamps: Vec<i64>,
3034 histograms: Vec<NativeHistogram>,
3035 range_end: i64,
3036 range_length: i64,
3037 ) -> Option<NativeHistogram> {
3038 assert_eq!(timestamps.len(), histograms.len());
3039 let range = [(0, u32::try_from(histograms.len()).unwrap())];
3040 let timestamps = Arc::new(TimestampMillisecondArray::from(timestamps));
3041 let histograms =
3042 build_histogram_array(&histograms.into_iter().map(Some).collect::<Vec<_>>());
3043 let mut input = vec![
3044 ColumnarValue::Array(Arc::new(
3045 RangeArray::from_ranges(timestamps, range)
3046 .unwrap()
3047 .into_dict(),
3048 )),
3049 ColumnarValue::Array(Arc::new(
3050 RangeArray::from_ranges(histograms, range)
3051 .unwrap()
3052 .into_dict(),
3053 )),
3054 ];
3055 input.push(ColumnarValue::Array(Arc::new(
3056 TimestampMillisecondArray::from(vec![range_end]),
3057 )));
3058 input.push(ColumnarValue::Array(Arc::new(Int64Array::from(vec![
3059 range_length,
3060 ]))));
3061 let result = run_udf(udf, input, native_histogram_arrow_type());
3062 let result = extract_array(&result).unwrap();
3063 let result = result.as_any().downcast_ref::<StructArray>().unwrap();
3064 read_histogram(result, 0).unwrap()
3065 }
3066
3067 fn collected_warnings(collector: &PromqlAnnotationCollector) -> Vec<String> {
3068 let mut warnings = Vec::new();
3069 collector.append_to(&mut warnings, &mut Vec::new());
3070 warnings
3071 }
3072
3073 fn collected_infos(collector: &PromqlAnnotationCollector) -> Vec<String> {
3074 let mut infos = Vec::new();
3075 collector.append_to(&mut Vec::new(), &mut infos);
3076 infos
3077 }
3078
3079 #[test]
3080 fn mixed_ranges_follow_prometheus_sample_type_semantics() {
3081 let first = sample_histogram(1.0, 1.0, vec![1.0]);
3082 let second = sample_histogram(3.0, 3.0, vec![3.0]);
3083 let collector = PromqlAnnotationCollector::default();
3084
3085 let mut rate = mixed_range_input(
3086 "rate",
3087 vec![Some(1.0), None, Some(3.0)],
3088 vec![None, Some(first.clone()), None],
3089 );
3090 rate.push(ColumnarValue::Array(Arc::new(
3091 TimestampMillisecondArray::from(vec![3000]),
3092 )));
3093 rate.push(ColumnarValue::Array(Arc::new(Int64Array::from(vec![3000]))));
3094 assert_eq!(
3095 mixed_float_result(MixedRange::float_udf(Some(collector.clone())), rate.clone()),
3096 None
3097 );
3098 assert_eq!(
3099 mixed_histogram_result(MixedRange::histogram_udf(Some(collector.clone())), rate),
3100 None
3101 );
3102 assert!(
3103 collected_warnings(&collector)
3104 .iter()
3105 .any(|warning| warning.contains("mix of float and native histogram"))
3106 );
3107
3108 let pure_rate = |floats, histograms| {
3109 let mut input = mixed_range_input("rate", floats, histograms);
3110 input.push(ColumnarValue::Array(Arc::new(
3111 TimestampMillisecondArray::from(vec![3000]),
3112 )));
3113 input.push(ColumnarValue::Array(Arc::new(Int64Array::from(vec![3000]))));
3114 input
3115 };
3116 let pure_float = pure_rate(
3117 vec![Some(1.0), Some(2.0), Some(3.0)],
3118 vec![None, None, None],
3119 );
3120 assert_eq!(
3121 mixed_float_result(MixedRange::float_udf(None), pure_float.clone()),
3122 Some(1.0)
3123 );
3124 assert_eq!(
3125 mixed_histogram_result(MixedRange::histogram_udf(None), pure_float),
3126 None
3127 );
3128 let pure_histogram = pure_rate(
3129 vec![None, None, None],
3130 vec![
3131 Some(sample_histogram(1.0, 1.0, vec![1.0])),
3132 Some(sample_histogram(2.0, 2.0, vec![2.0])),
3133 Some(sample_histogram(3.0, 3.0, vec![3.0])),
3134 ],
3135 );
3136 assert_eq!(
3137 mixed_float_result(MixedRange::float_udf(None), pure_histogram.clone()),
3138 None
3139 );
3140 assert_eq!(
3141 mixed_histogram_result(MixedRange::histogram_udf(None), pure_histogram)
3142 .unwrap()
3143 .count,
3144 1.0
3145 );
3146
3147 let idelta = mixed_range_input(
3148 "idelta",
3149 vec![Some(10.0), None, None],
3150 vec![None, Some(first.clone()), Some(second.clone())],
3151 );
3152 assert_eq!(
3153 mixed_float_result(MixedRange::float_udf(None), idelta.clone()),
3154 None
3155 );
3156 assert_eq!(
3157 mixed_histogram_result(MixedRange::histogram_udf(None), idelta)
3158 .unwrap()
3159 .count,
3160 2.0
3161 );
3162
3163 let alternating = || {
3164 mixed_range_input(
3165 "changes",
3166 vec![Some(1.0), None, None, Some(1.0)],
3167 vec![None, Some(first.clone()), Some(first.clone()), None],
3168 )
3169 };
3170 assert_eq!(
3171 mixed_float_result(MixedRange::float_udf(None), alternating()),
3172 Some(2.0)
3173 );
3174 let mut resets = alternating();
3175 resets[0] = ColumnarValue::Scalar(ScalarValue::Utf8(Some("resets".to_string())));
3176 assert_eq!(
3177 mixed_float_result(MixedRange::float_udf(None), resets),
3178 Some(2.0)
3179 );
3180
3181 for (name, expected) in [
3182 ("count_over_time", Some(4.0)),
3183 ("present_over_time", Some(1.0)),
3184 ("absent_over_time", None),
3185 ] {
3186 let mut input = alternating();
3187 input[0] = ColumnarValue::Scalar(ScalarValue::Utf8(Some(name.to_string())));
3188 assert_eq!(
3189 mixed_float_result(MixedRange::float_udf(None), input),
3190 expected,
3191 "{name}"
3192 );
3193 }
3194
3195 let last = mixed_range_input(
3196 "last_over_time",
3197 vec![None, Some(4.0)],
3198 vec![Some(first.clone()), None],
3199 );
3200 assert_eq!(
3201 mixed_float_result(MixedRange::float_udf(None), last.clone()),
3202 Some(4.0)
3203 );
3204 assert_eq!(
3205 mixed_histogram_result(MixedRange::histogram_udf(None), last),
3206 None
3207 );
3208
3209 let collector = PromqlAnnotationCollector::default();
3210 let histogram_only_min =
3211 mixed_range_input("min_over_time", vec![None], vec![Some(first.clone())]);
3212 assert_eq!(
3213 mixed_float_result(
3214 MixedRange::float_udf(Some(collector.clone())),
3215 histogram_only_min,
3216 ),
3217 None
3218 );
3219 assert!(collected_infos(&collector).is_empty());
3220
3221 let min = mixed_range_input(
3222 "min_over_time",
3223 vec![Some(3.0), None, Some(1.0)],
3224 vec![None, Some(first), None],
3225 );
3226 assert_eq!(
3227 mixed_float_result(MixedRange::float_udf(Some(collector.clone())), min),
3228 Some(1.0)
3229 );
3230 assert!(
3231 collected_infos(&collector)
3232 .iter()
3233 .any(|info| info.contains("ignored native histogram"))
3234 );
3235 }
3236
3237 #[test]
3238 fn count_sum_and_avg_read_struct() {
3239 let histograms = vec![Some(sample_histogram(6.0, 10.0, vec![2.0, 4.0]))];
3240 let array = build_histogram_array(&histograms);
3241 let input = vec![ColumnarValue::Array(array)];
3242
3243 let count = run_scalar_udf(NativeHistogramCount::scalar_udf(), input.clone());
3244 assert_eq!(count, 6.0);
3245
3246 let sum = run_scalar_udf(NativeHistogramSum::scalar_udf(), input.clone());
3247 assert_eq!(sum, 10.0);
3248
3249 let avg = run_scalar_udf(NativeHistogramAvg::scalar_udf(), input);
3250 assert_eq!(avg, 10.0 / 6.0);
3251 }
3252
3253 #[test]
3254 fn quantile_uses_bucket_bounds() {
3255 let histogram = sample_histogram(6.0, 10.0, vec![2.0, 4.0]);
3256 assert_eq!(histogram.quantile(0.0), 0.5);
3257 assert!(histogram.quantile(0.5) > 1.0);
3258 assert!(histogram.quantile(0.5) < 2.0);
3259 }
3260
3261 #[test]
3262 fn comparison_observes_explicit_sparse_zero_buckets() {
3263 let mut left = sample_histogram(1.0, 1.0, vec![1.0, 0.0]);
3264 left.reset_hint = COUNTER_RESET_HINT;
3265 left.start_timestamp = Some(1000);
3266 let mut right = sample_histogram(1.0, 1.0, vec![1.0]);
3267 right.reset_hint = NOT_COUNTER_RESET_HINT;
3268 right.start_timestamp = Some(2000);
3269
3270 let result = run_udf(
3271 NativeHistogramEq::scalar_udf(),
3272 vec![
3273 ColumnarValue::Array(build_histogram_array(&[Some(left)])),
3274 ColumnarValue::Array(build_histogram_array(&[Some(right)])),
3275 ],
3276 DataType::Boolean,
3277 );
3278 let values = extract_array(&result).unwrap();
3279 let values = values.as_any().downcast_ref::<BooleanArray>().unwrap();
3280 assert!(!values.value(0));
3281 }
3282
3283 #[test]
3284 fn unary_minus_returns_gauge_histogram() {
3285 let result = run_histogram_udf(
3286 NativeHistogramNeg::scalar_udf(),
3287 vec![ColumnarValue::Array(build_histogram_array(&[Some(
3288 sample_histogram(2.0, 3.0, vec![2.0]),
3289 )]))],
3290 );
3291
3292 assert_eq!(result.reset_hint, GAUGE_RESET_HINT);
3293 assert_eq!(result.count, -2.0);
3294 assert_eq!(result.sum, -3.0);
3295 assert_eq!(result.positive_buckets, vec![-2.0]);
3296 }
3297
3298 #[test]
3299 fn histogram_over_time_functions_preserve_reset_hints() {
3300 let mut first = sample_histogram(1.0, 1.0, vec![1.0]);
3301 first.reset_hint = COUNTER_RESET_HINT;
3302 let mut second = sample_histogram(2.0, 2.0, vec![2.0]);
3303 second.reset_hint = COUNTER_RESET_HINT;
3304
3305 let result = run_histogram_range_udf(
3306 NativeHistogramSumOverTime::scalar_udf(),
3307 vec![first.clone(), second.clone()],
3308 );
3309 assert_eq!(result.reset_hint, COUNTER_RESET_HINT);
3310 assert_eq!(result.count, 3.0);
3311 assert_eq!(result.sum, 3.0);
3312 assert_eq!(result.positive_buckets, vec![3.0]);
3313
3314 let result = run_histogram_range_udf(
3315 NativeHistogramAvgOverTime::scalar_udf(),
3316 vec![first.clone(), second.clone()],
3317 );
3318 assert_eq!(result.reset_hint, COUNTER_RESET_HINT);
3319 assert_eq!(result.count, 1.5);
3320 assert_eq!(result.sum, 1.5);
3321 assert_eq!(result.positive_buckets, vec![1.5]);
3322
3323 let result = run_histogram_range_udf(
3324 NativeHistogramLastOverTime::scalar_udf(),
3325 vec![first, second],
3326 );
3327 assert_eq!(result.reset_hint, COUNTER_RESET_HINT);
3328 assert_eq!(result.count, 2.0);
3329 assert_eq!(result.sum, 2.0);
3330 assert_eq!(result.positive_buckets, vec![2.0]);
3331 }
3332
3333 #[test]
3334 fn histogram_averages_avoid_sum_overflow() {
3335 let large = sample_histogram(1.0e308, 1.0e308, vec![1.0e308]);
3336
3337 let range_average = run_histogram_range_udf(
3338 NativeHistogramAvgOverTime::scalar_udf(),
3339 vec![large.clone(), large.clone()],
3340 );
3341 assert_eq!(range_average.count, 1.0e308);
3342 assert_eq!(range_average.sum, 1.0e308);
3343 assert_eq!(range_average.positive_buckets, vec![1.0e308]);
3344
3345 let mut aggregate =
3346 NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3347 aggregate.push_histogram(large.clone(), 1).unwrap();
3348 aggregate.push_histogram(large, 1).unwrap();
3349 let aggregate_average = evaluated_histogram(&mut aggregate);
3350 assert_eq!(aggregate_average.count, 1.0e308);
3351 assert_eq!(aggregate_average.sum, 1.0e308);
3352 assert_eq!(aggregate_average.positive_buckets, vec![1.0e308]);
3353 }
3354
3355 #[test]
3356 fn histogram_average_partial_states_are_weighted() {
3357 let mut first =
3358 NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3359 first
3360 .push_histogram(sample_histogram(1.0, 1.0, vec![1.0]), 1)
3361 .unwrap();
3362 first
3363 .push_histogram(sample_histogram(3.0, 3.0, vec![3.0]), 1)
3364 .unwrap();
3365 let first_state = first
3366 .state()
3367 .unwrap()
3368 .into_iter()
3369 .map(|value| value.to_array_of_size(1).unwrap())
3370 .collect::<Vec<_>>();
3371
3372 let mut second =
3373 NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3374 second
3375 .push_histogram(sample_histogram(8.0, 8.0, vec![8.0]), 1)
3376 .unwrap();
3377 let second_state = second
3378 .state()
3379 .unwrap()
3380 .into_iter()
3381 .map(|value| value.to_array_of_size(1).unwrap())
3382 .collect::<Vec<_>>();
3383
3384 let mut merged =
3385 NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3386 merged.merge_batch(&first_state).unwrap();
3387 merged.merge_batch(&second_state).unwrap();
3388 let average = evaluated_histogram(&mut merged);
3389 assert_eq!(average.count, 4.0);
3390 assert_eq!(average.sum, 4.0);
3391 assert_eq!(average.positive_buckets, vec![4.0]);
3392 }
3393
3394 #[test]
3395 fn histogram_average_rejects_sample_count_overflow() {
3396 let mut aggregate =
3397 NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3398 aggregate.value = Some(sample_histogram(1.0, 1.0, vec![1.0]));
3399 aggregate.count = u64::MAX;
3400
3401 let error = aggregate
3402 .push_histogram(sample_histogram(1.0, 1.0, vec![1.0]), 1)
3403 .unwrap_err();
3404 assert!(error.to_string().contains("sample count overflow"));
3405 }
3406
3407 #[test]
3408 fn presence_only_range_functions_preserve_null_semantics() {
3409 let first = sample_histogram(1.0, 1.0, vec![1.0]);
3410 let second = sample_histogram(2.0, 2.0, vec![2.0]);
3411 assert_eq!(
3412 run_float_range_udf(
3413 NativeHistogramCountOverTime::scalar_udf(),
3414 vec![Some(first.clone()), Some(second.clone())],
3415 ),
3416 Some(2.0)
3417 );
3418 assert_eq!(
3419 run_float_range_udf(
3420 NativeHistogramPresentOverTime::scalar_udf(),
3421 vec![Some(first.clone()), Some(second.clone())],
3422 ),
3423 Some(1.0)
3424 );
3425 assert_eq!(
3426 run_float_range_udf(
3427 NativeHistogramCountOverTime::scalar_udf(),
3428 vec![None, Some(second.clone())],
3429 ),
3430 None
3431 );
3432
3433 let result = run_udf(
3434 NativeHistogramLastOverTime::scalar_udf(),
3435 histogram_range_input(vec![None, Some(second)]),
3436 native_histogram_arrow_type(),
3437 );
3438 let result = extract_array(&result).unwrap();
3439 let result = result.as_any().downcast_ref::<StructArray>().unwrap();
3440 assert!(result.is_null(0));
3441 }
3442
3443 #[test]
3444 fn resets_counts_histogram_flavor_transitions() {
3445 let mut counter = sample_histogram(1.0, 1.0, vec![1.0]);
3446 counter.reset_hint = NOT_COUNTER_RESET_HINT;
3447 let mut gauge = sample_histogram(2.0, 2.0, vec![2.0]);
3448 gauge.reset_hint = GAUGE_RESET_HINT;
3449 assert_eq!(
3450 run_float_range_udf(
3451 NativeHistogramResets::scalar_udf(),
3452 vec![Some(counter), Some(gauge)],
3453 ),
3454 Some(1.0)
3455 );
3456
3457 let mut gauge = sample_histogram(1.0, 1.0, vec![1.0]);
3458 gauge.reset_hint = GAUGE_RESET_HINT;
3459 let mut counter = sample_histogram(2.0, 2.0, vec![2.0]);
3460 counter.reset_hint = NOT_COUNTER_RESET_HINT;
3461 assert_eq!(
3462 run_float_range_udf(
3463 NativeHistogramResets::scalar_udf(),
3464 vec![Some(gauge), Some(counter)],
3465 ),
3466 Some(1.0)
3467 );
3468 }
3469
3470 #[test]
3471 fn wrong_flavor_functions_record_warnings() {
3472 let mut gauge_first = sample_histogram(1.0, 1.0, vec![1.0]);
3473 gauge_first.reset_hint = GAUGE_RESET_HINT;
3474 let mut gauge_last = sample_histogram(2.0, 2.0, vec![2.0]);
3475 gauge_last.reset_hint = GAUGE_RESET_HINT;
3476 let mut counter_first = sample_histogram(1.0, 1.0, vec![1.0]);
3477 counter_first.reset_hint = NOT_COUNTER_RESET_HINT;
3478 let mut counter_last = sample_histogram(2.0, 2.0, vec![2.0]);
3479 counter_last.reset_hint = NOT_COUNTER_RESET_HINT;
3480 let collector = PromqlAnnotationCollector::default();
3481
3482 run_extrapolated_histogram_udf(
3483 NativeHistogramRate::scalar_udf_with_collector(Some(collector.clone())),
3484 vec![gauge_first.clone(), gauge_last.clone()],
3485 );
3486 run_extrapolated_histogram_udf(
3487 NativeHistogramDelta::scalar_udf_with_collector(Some(collector.clone())),
3488 vec![counter_first.clone(), counter_last.clone()],
3489 );
3490 run_histogram_range_udf(
3491 NativeHistogramIRate::scalar_udf_with_collector(Some(collector.clone())),
3492 vec![gauge_first, gauge_last],
3493 );
3494 run_histogram_range_udf(
3495 NativeHistogramIDelta::scalar_udf_with_collector(Some(collector.clone())),
3496 vec![counter_first, counter_last],
3497 );
3498
3499 let warnings = collected_warnings(&collector);
3500 for expected in [
3501 format!(
3502 "{}: native histogram input should be a counter histogram",
3503 NativeHistogramRate::name()
3504 ),
3505 format!(
3506 "{}: native histogram input should be a gauge histogram",
3507 NativeHistogramDelta::name()
3508 ),
3509 format!(
3510 "{}: native histogram input should be a counter histogram",
3511 NativeHistogramIRate::name()
3512 ),
3513 format!(
3514 "{}: native histogram input should be a gauge histogram",
3515 NativeHistogramIDelta::name()
3516 ),
3517 ] {
3518 assert!(warnings.contains(&expected), "missing warning: {expected}");
3519 }
3520 }
3521
3522 #[test]
3523 fn subtraction_records_reset_hint_contradictions_for_incompatible_histograms() {
3524 for incompatible in [false, true] {
3525 let mut left = sample_histogram(2.0, 2.0, vec![2.0]);
3526 left.reset_hint = COUNTER_RESET_HINT;
3527 let mut right = sample_histogram(1.0, 1.0, vec![1.0]);
3528 right.reset_hint = NOT_COUNTER_RESET_HINT;
3529 if incompatible {
3530 right.schema = CUSTOM_BUCKETS_SCHEMA;
3531 right.custom_values = vec![1.0];
3532 }
3533 let collector = PromqlAnnotationCollector::default();
3534
3535 run_udf(
3536 NativeHistogramSub::scalar_udf_with_collector(Some(collector.clone())),
3537 vec![
3538 ColumnarValue::Array(build_histogram_array(&[Some(left)])),
3539 ColumnarValue::Array(build_histogram_array(&[Some(right)])),
3540 ],
3541 native_histogram_arrow_type(),
3542 );
3543
3544 assert!(collected_warnings(&collector).contains(&format!(
3545 "{}: native histogram counter reset hints contradict",
3546 NativeHistogramSub::name()
3547 )));
3548 }
3549 }
3550
3551 #[test]
3552 fn counter_reset_hint_history_survives_folds_and_state_merges() {
3553 let mut reset = sample_histogram(1.0, 1.0, vec![1.0]);
3554 reset.reset_hint = COUNTER_RESET_HINT;
3555 let mut unknown = sample_histogram(2.0, 2.0, vec![2.0]);
3556 unknown.reset_hint = UNKNOWN_COUNTER_RESET_HINT;
3557 let mut not_reset = sample_histogram(3.0, 3.0, vec![3.0]);
3558 not_reset.reset_hint = NOT_COUNTER_RESET_HINT;
3559
3560 let range_collector = PromqlAnnotationCollector::default();
3561 assert!(
3562 range_fold_histograms(
3563 vec![reset.clone(), unknown.clone(), not_reset.clone()],
3564 NativeHistogramAggregateKind::Sum,
3565 NativeHistogramSumOverTime::name(),
3566 &Some(range_collector.clone()),
3567 )
3568 .is_some()
3569 );
3570 assert!(collected_warnings(&range_collector).contains(&format!(
3571 "{}: native histogram counter reset hints contradict",
3572 NativeHistogramSumOverTime::name()
3573 )));
3574
3575 for kind in [
3576 NativeHistogramAggregateKind::Sum,
3577 NativeHistogramAggregateKind::Avg,
3578 ] {
3579 let mut first_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3580 first_partial.push_histogram(reset.clone(), 1).unwrap();
3581 first_partial.push_histogram(unknown.clone(), 1).unwrap();
3582 let first_states = first_partial
3583 .state()
3584 .unwrap()
3585 .into_iter()
3586 .map(|value| value.to_array_of_size(1).unwrap())
3587 .collect::<Vec<_>>();
3588
3589 let mut second_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3590 second_partial.push_histogram(not_reset.clone(), 1).unwrap();
3591 let second_states = second_partial
3592 .state()
3593 .unwrap()
3594 .into_iter()
3595 .map(|value| value.to_array_of_size(1).unwrap())
3596 .collect::<Vec<_>>();
3597
3598 let collector = PromqlAnnotationCollector::default();
3599 let mut merged =
3600 NativeHistogramAggregateAccumulator::new(kind, Some(collector.clone()));
3601 merged.merge_batch(&first_states).unwrap();
3602 merged.merge_batch(&second_states).unwrap();
3603 assert!(collected_warnings(&collector).contains(&format!(
3604 "{}: native histogram counter reset hints contradict",
3605 kind.name()
3606 )));
3607 }
3608 }
3609
3610 #[test]
3611 fn incompatible_aggregates_do_not_hide_reset_hint_contradictions() {
3612 let mut reset = sample_histogram(1.0, 1.0, vec![1.0]);
3613 reset.reset_hint = COUNTER_RESET_HINT;
3614 let mut incompatible_reset = sample_histogram(2.0, 2.0, vec![2.0]);
3615 incompatible_reset.schema = CUSTOM_BUCKETS_SCHEMA;
3616 incompatible_reset.custom_values = vec![1.0];
3617 incompatible_reset.reset_hint = COUNTER_RESET_HINT;
3618 let mut not_reset = sample_histogram(3.0, 3.0, vec![3.0]);
3619 not_reset.reset_hint = NOT_COUNTER_RESET_HINT;
3620
3621 for kind in [
3622 NativeHistogramAggregateKind::Sum,
3623 NativeHistogramAggregateKind::Avg,
3624 ] {
3625 for histograms in [
3626 [reset.clone(), incompatible_reset.clone(), not_reset.clone()],
3627 [reset.clone(), not_reset.clone(), incompatible_reset.clone()],
3628 ] {
3629 let collector = PromqlAnnotationCollector::default();
3630 let mut aggregate =
3631 NativeHistogramAggregateAccumulator::new(kind, Some(collector.clone()));
3632 for histogram in histograms {
3633 aggregate.push_histogram(histogram, 1).unwrap();
3634 }
3635
3636 assert!(aggregate.dropped_incompatible);
3637 let warnings = collected_warnings(&collector);
3638 assert!(warnings.contains(&format!(
3639 "{}: dropped native histogram aggregate with incompatible schemas",
3640 kind.name()
3641 )));
3642 assert!(warnings.contains(&format!(
3643 "{}: native histogram counter reset hints contradict",
3644 kind.name()
3645 )));
3646 }
3647
3648 let mut dropped_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3649 dropped_partial.push_histogram(reset.clone(), 1).unwrap();
3650 dropped_partial
3651 .push_histogram(incompatible_reset.clone(), 1)
3652 .unwrap();
3653 let dropped_states = dropped_partial
3654 .state()
3655 .unwrap()
3656 .into_iter()
3657 .map(|value| value.to_array_of_size(1).unwrap())
3658 .collect::<Vec<_>>();
3659
3660 let mut opposing_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3661 opposing_partial
3662 .push_histogram(not_reset.clone(), 1)
3663 .unwrap();
3664 let opposing_states = opposing_partial
3665 .state()
3666 .unwrap()
3667 .into_iter()
3668 .map(|value| value.to_array_of_size(1).unwrap())
3669 .collect::<Vec<_>>();
3670
3671 for (first, second) in [
3672 (&dropped_states, &opposing_states),
3673 (&opposing_states, &dropped_states),
3674 ] {
3675 let collector = PromqlAnnotationCollector::default();
3676 let mut merged =
3677 NativeHistogramAggregateAccumulator::new(kind, Some(collector.clone()));
3678 merged.merge_batch(first).unwrap();
3679 merged.merge_batch(second).unwrap();
3680
3681 assert!(merged.dropped_incompatible);
3682 assert!(collected_warnings(&collector).contains(&format!(
3683 "{}: native histogram counter reset hints contradict",
3684 kind.name()
3685 )));
3686 }
3687 }
3688
3689 for (kind, name) in [
3690 (
3691 NativeHistogramAggregateKind::Sum,
3692 NativeHistogramSumOverTime::name(),
3693 ),
3694 (
3695 NativeHistogramAggregateKind::Avg,
3696 NativeHistogramAvgOverTime::name(),
3697 ),
3698 ] {
3699 let collector = PromqlAnnotationCollector::default();
3700 assert!(
3701 range_fold_histograms(
3702 vec![reset.clone(), incompatible_reset.clone(), not_reset.clone()],
3703 kind,
3704 name,
3705 &Some(collector.clone()),
3706 )
3707 .is_none()
3708 );
3709 let warnings = collected_warnings(&collector);
3710 assert!(warnings.contains(&format!(
3711 "{name}: dropped native histogram range with incompatible schemas"
3712 )));
3713 assert!(warnings.contains(&format!(
3714 "{name}: native histogram counter reset hints contradict"
3715 )));
3716 }
3717 }
3718
3719 #[test]
3720 fn histogram_aggregate_accumulator_accounts_for_heap_allocations() {
3721 let histogram = sample_histogram(3.0, 3.0, vec![1.0, 2.0]);
3722 let heap_size = histogram.custom_values.capacity() * size_of::<f64>()
3723 + histogram.positive_spans.capacity() * size_of::<Span>()
3724 + histogram.negative_spans.capacity() * size_of::<Span>()
3725 + histogram.positive_buckets.capacity() * size_of::<f64>()
3726 + histogram.negative_buckets.capacity() * size_of::<f64>();
3727 let mut accumulator =
3728 NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Sum, None);
3729 let empty_size = accumulator.size();
3730
3731 accumulator.push_histogram(histogram, 1).unwrap();
3732
3733 assert!(heap_size > 0);
3734 assert_eq!(accumulator.size(), empty_size + heap_size);
3735 }
3736
3737 #[test]
3738 fn absent_over_time_handles_histogram_ranges() {
3739 let values = vec![Some(sample_histogram(1.0, 1.0, vec![1.0]))];
3740 let timestamps = Arc::new(TimestampMillisecondArray::from_iter([Some(1000)]));
3741 let histograms = build_histogram_array(&values);
3742 let ranges = [(0, 1), (0, 0)];
3743 let ts_range = RangeArray::from_ranges(timestamps, ranges).unwrap();
3744 let value_range = RangeArray::from_ranges(histograms, ranges).unwrap();
3745
3746 let result = run_udf(
3747 NativeHistogramAbsentOverTime::scalar_udf(),
3748 vec![
3749 ColumnarValue::Array(Arc::new(ts_range.into_dict())),
3750 ColumnarValue::Array(Arc::new(value_range.into_dict())),
3751 ],
3752 DataType::Float64,
3753 );
3754 let result = extract_array(&result).unwrap();
3755 let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
3756
3757 assert!(result.is_null(0));
3758 assert_eq!(result.value(1), 1.0);
3759 }
3760
3761 #[test]
3762 fn delta_requires_exact_layout() {
3763 let first = sample_histogram(2.0, 3.0, vec![1.0, 1.0]);
3764 let last = sample_histogram(5.0, 8.0, vec![2.0, 3.0]);
3765 let delta = histogram_delta(&[first, last], &[0, 1], false).unwrap();
3766 assert_eq!(delta.count, 3.0);
3767 assert_eq!(delta.sum, 5.0);
3768 assert_eq!(delta.reset_hint, GAUGE_RESET_HINT);
3769 assert_eq!(delta.positive_buckets, vec![1.0, 2.0]);
3770 }
3771
3772 #[test]
3773 fn reset_hint_shortcuts_detection() {
3774 let previous = sample_histogram(6.0, 10.0, vec![2.0, 4.0]);
3775
3776 let mut current = sample_histogram(7.0, 12.0, vec![3.0, 4.0]);
3777 current.reset_hint = COUNTER_RESET_HINT;
3778 assert!(current.detect_reset(&previous));
3779
3780 let mut current = sample_histogram(5.0, 8.0, vec![1.0, 4.0]);
3781 current.reset_hint = NOT_COUNTER_RESET_HINT;
3782 assert!(!current.detect_reset(&previous));
3783 }
3784
3785 #[test]
3786 fn start_timestamp_detects_counter_reset() {
3787 let first = sample_histogram(6.0, 10.0, vec![2.0, 4.0]);
3788 let mut last = sample_histogram(7.0, 12.0, vec![3.0, 4.0]);
3789 last.start_timestamp = Some(1500);
3790
3791 let delta = histogram_delta(&[first.clone(), last.clone()], &[1000, 2000], true).unwrap();
3792 assert_eq!(delta.count, 7.0);
3793 assert_eq!(delta.sum, 12.0);
3794
3795 let idelta = idelta_value(&[first, last], true, 1000, 2000, 1.0).unwrap();
3796 assert_eq!(idelta.count, 7.0);
3797 assert_eq!(idelta.sum, 12.0);
3798 }
3799
3800 #[test]
3801 fn extrapolated_rate_uses_start_timestamp_synthetic_zero() {
3802 let mut single = sample_histogram(1.0, 1.0, vec![1.0]);
3803 single.start_timestamp = Some(1_000);
3804
3805 let rate = extrapolated_histogram_result(
3806 NativeHistogramRate::scalar_udf(),
3807 vec![2_000],
3808 vec![single.clone()],
3809 3_000,
3810 3_000,
3811 )
3812 .unwrap();
3813 assert_eq!(rate.count, 1.0 / 3.0);
3814 assert_eq!(rate.sum, 1.0 / 3.0);
3815
3816 let increase = extrapolated_histogram_result(
3817 NativeHistogramIncrease::scalar_udf(),
3818 vec![2_000],
3819 vec![single],
3820 3_000,
3821 3_000,
3822 )
3823 .unwrap();
3824 assert_eq!(increase.count, 1.0);
3825 assert_eq!(increase.sum, 1.0);
3826
3827 let mut first = sample_histogram(2.0, 2.0, vec![2.0]);
3828 first.start_timestamp = Some(1_000);
3829 let last = sample_histogram(4.0, 4.0, vec![4.0]);
3830 let increase = extrapolated_histogram_result(
3831 NativeHistogramIncrease::scalar_udf(),
3832 vec![2_000, 3_000],
3833 vec![first, last],
3834 3_000,
3835 3_000,
3836 )
3837 .unwrap();
3838 assert_eq!(increase.count, 4.0);
3839 assert_eq!(increase.sum, 4.0);
3840 }
3841
3842 #[test]
3843 fn extrapolated_rate_requires_strictly_in_range_start_timestamp() {
3844 for (start_timestamp, range_end, range_length) in [
3845 (0, 3_000, 3_000),
3846 (500, 3_000, 2_000),
3847 (1_000, 3_000, 2_000),
3848 (2_000, 3_000, 3_000),
3849 (2_500, 3_000, 3_000),
3850 ] {
3851 let mut sample = sample_histogram(1.0, 1.0, vec![1.0]);
3852 sample.start_timestamp = Some(start_timestamp);
3853 assert!(
3854 extrapolated_histogram_result(
3855 NativeHistogramRate::scalar_udf(),
3856 vec![2_000],
3857 vec![sample],
3858 range_end,
3859 range_length,
3860 )
3861 .is_none(),
3862 "start_timestamp={start_timestamp}"
3863 );
3864 }
3865
3866 let mut gauge = sample_histogram(1.0, 1.0, vec![1.0]);
3867 gauge.start_timestamp = Some(1_000);
3868 gauge.reset_hint = GAUGE_RESET_HINT;
3869 assert!(
3870 extrapolated_histogram_result(
3871 NativeHistogramDelta::scalar_udf(),
3872 vec![2_000],
3873 vec![gauge],
3874 3_000,
3875 3_000,
3876 )
3877 .is_none()
3878 );
3879 }
3880
3881 #[test]
3882 fn first_reset_ignores_incompatible_pre_reset_layout() {
3883 let first = sample_histogram(10.0, 10.0, vec![10.0]);
3884 let second = NativeHistogram {
3885 schema: CUSTOM_BUCKETS_SCHEMA,
3886 zero_threshold: 0.0,
3887 sum: 2.0,
3888 reset_hint: COUNTER_RESET_HINT,
3889 start_timestamp: None,
3890 custom_values: vec![1.0],
3891 positive_spans: vec![Span {
3892 offset: 0,
3893 length: 1,
3894 }],
3895 negative_spans: Vec::new(),
3896 count: 2.0,
3897 zero_count: 0.0,
3898 positive_buckets: vec![2.0],
3899 negative_buckets: Vec::new(),
3900 };
3901
3902 let delta = histogram_delta(&[first, second], &[1_000, 2_000], true).unwrap();
3903 assert_eq!(delta.schema, CUSTOM_BUCKETS_SCHEMA);
3904 assert_eq!(delta.count, 2.0);
3905 assert_eq!(delta.sum, 2.0);
3906 assert_eq!(delta.positive_buckets, vec![2.0]);
3907 }
3908
3909 #[test]
3910 fn counter_delta_handles_reset_segment_boundaries() {
3911 let first = sample_histogram(5.0, 5.0, vec![5.0]);
3912 let second = sample_histogram(7.0, 7.0, vec![7.0]);
3913 let mut reset = sample_histogram(2.0, 2.0, vec![2.0]);
3914 reset.reset_hint = COUNTER_RESET_HINT;
3915 let last = sample_histogram(4.0, 4.0, vec![4.0]);
3916
3917 let delta = histogram_delta(
3918 &[first, second, reset, last],
3919 &[1_000, 2_000, 3_000, 4_000],
3920 true,
3921 )
3922 .unwrap();
3923 assert_eq!(delta.count, 6.0);
3924 assert_eq!(delta.sum, 6.0);
3925 assert_eq!(delta.positive_buckets, vec![6.0]);
3926 }
3927}