Skip to main content

promql/functions/
native_histogram.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Native histogram PromQL helpers.
16
17use std::any::Any;
18use std::hash::{Hash, Hasher};
19use std::mem::size_of;
20use std::sync::Arc;
21
22use common_query::native_histogram::*;
23use common_query::prometheus::format_prometheus_float;
24use common_query::promql_annotations::PromqlAnnotationCollector;
25use datafusion::arrow::array::{
26    Array, ArrayRef, BooleanArray, Float64Array, Float64Builder, Int64Array, StringBuilder,
27    StructArray, TimestampMillisecondArray, UInt64Array,
28};
29use datafusion::arrow::compute::filter;
30use datafusion::arrow::datatypes::{DataType, Field, TimeUnit};
31use datafusion::common::{DataFusionError, Result as DfResult};
32use datafusion::logical_expr::{Accumulator as DfAccumulator, AggregateUDF, ScalarUDF, Volatility};
33use datafusion::physical_plan::ColumnarValue;
34use datafusion_common::ScalarValue;
35use datafusion_expr::function::AccumulatorArgs;
36use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, create_udaf, create_udf};
37
38use crate::functions::{
39    AvgOverTime, Deriv, DoubleExponentialSmoothing, IDelta, Increase, LastOverTime, MaxOverTime,
40    MinOverTime, PredictLinear, QuantileOverTime, Rate, StddevOverTime, StdvarOverTime,
41    SumOverTime, extract_array, extract_range_dict,
42};
43use crate::range_array::{RangeArray, unpack};
44
45fn extract_histogram_array(value: &ColumnarValue, func_name: &str) -> DfResult<ArrayRef> {
46    let array = extract_array(value)?;
47    if array.data_type() != &native_histogram_arrow_type() {
48        return Err(DataFusionError::Execution(format!(
49            "{func_name}: expected native histogram struct, found {}",
50            array.data_type()
51        )));
52    }
53    Ok(array)
54}
55
56fn read_scalar_f64_arg(
57    value: &ColumnarValue,
58    row: usize,
59    len: usize,
60    func_name: &str,
61) -> DfResult<f64> {
62    match value {
63        ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok(value.unwrap_or(f64::NAN)),
64        ColumnarValue::Array(array) => {
65            let array = array
66                .as_any()
67                .downcast_ref::<Float64Array>()
68                .ok_or_else(|| {
69                    DataFusionError::Execution(format!(
70                        "{func_name}: expected Float64 argument, found {}",
71                        array.data_type()
72                    ))
73                })?;
74            if array.len() != len {
75                return Err(DataFusionError::Execution(format!(
76                    "{func_name}: Float64 argument length mismatch: {} vs {len}",
77                    array.len()
78                )));
79            }
80            Ok(if array.is_null(row) {
81                f64::NAN
82            } else {
83                array.value(row)
84            })
85        }
86        other => Err(DataFusionError::Execution(format!(
87            "{func_name}: expected Float64 argument, found {}",
88            other.data_type()
89        ))),
90    }
91}
92
93#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
94enum AnnotationReturn {
95    FloatNull,
96    BooleanTrue,
97    BooleanFalse,
98}
99
100#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
101enum AnnotationLevel {
102    Info,
103    Warning,
104}
105
106impl AnnotationReturn {
107    fn data_type(self) -> DataType {
108        match self {
109            Self::FloatNull => DataType::Float64,
110            Self::BooleanTrue | Self::BooleanFalse => DataType::Boolean,
111        }
112    }
113
114    fn scalar_value(self) -> ScalarValue {
115        match self {
116            Self::FloatNull => ScalarValue::Float64(None),
117            Self::BooleanTrue => ScalarValue::Boolean(Some(true)),
118            Self::BooleanFalse => ScalarValue::Boolean(Some(false)),
119        }
120    }
121}
122
123#[derive(Debug, Clone)]
124struct NativeHistogramAnnotationUdf {
125    name: &'static str,
126    signature: Signature,
127    return_kind: AnnotationReturn,
128    level: AnnotationLevel,
129    message: String,
130    collector: Option<PromqlAnnotationCollector>,
131}
132
133impl NativeHistogramAnnotationUdf {
134    fn new(
135        name: &'static str,
136        return_kind: AnnotationReturn,
137        level: AnnotationLevel,
138        message: String,
139        collector: Option<PromqlAnnotationCollector>,
140    ) -> Self {
141        Self {
142            name,
143            signature: Signature::variadic_any(Volatility::Volatile),
144            return_kind,
145            level,
146            message,
147            collector,
148        }
149    }
150}
151
152impl PartialEq for NativeHistogramAnnotationUdf {
153    fn eq(&self, other: &Self) -> bool {
154        self.name == other.name
155            && self.return_kind == other.return_kind
156            && self.level == other.level
157            && self.message == other.message
158    }
159}
160
161impl Eq for NativeHistogramAnnotationUdf {}
162
163impl Hash for NativeHistogramAnnotationUdf {
164    fn hash<H: Hasher>(&self, state: &mut H) {
165        self.name.hash(state);
166        self.return_kind.hash(state);
167        self.level.hash(state);
168        self.message.hash(state);
169    }
170}
171
172impl ScalarUDFImpl for NativeHistogramAnnotationUdf {
173    fn as_any(&self) -> &dyn Any {
174        self
175    }
176
177    fn name(&self) -> &str {
178        self.name
179    }
180
181    fn signature(&self) -> &Signature {
182        &self.signature
183    }
184
185    fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
186        Ok(self.return_kind.data_type())
187    }
188
189    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult<ColumnarValue> {
190        let has_dropped_sample = !args.args.is_empty()
191            && (0..args.number_rows).any(|row| {
192                args.args.iter().all(|arg| match arg {
193                    ColumnarValue::Array(array) => array.is_valid(row),
194                    ColumnarValue::Scalar(value) => !value.is_null(),
195                })
196            });
197        if has_dropped_sample
198            && let Some(collector) = args
199                .config_options
200                .extensions
201                .get::<PromqlAnnotationCollector>()
202                .cloned()
203                .or_else(|| self.collector.clone())
204        {
205            match self.level {
206                AnnotationLevel::Info => collector.record_info(self.message.clone()),
207                AnnotationLevel::Warning => collector.record_warning(self.message.clone()),
208            }
209        }
210        Ok(ColumnarValue::Scalar(self.return_kind.scalar_value()))
211    }
212}
213
214pub struct NativeHistogramDrop;
215
216impl NativeHistogramDrop {
217    const fn float_null_name() -> &'static str {
218        "prom_native_histogram_drop_float"
219    }
220
221    const fn bool_false_name() -> &'static str {
222        "prom_native_histogram_drop_bool"
223    }
224
225    const fn bool_true_name() -> &'static str {
226        "prom_native_histogram_keep_bool"
227    }
228
229    pub fn float_null_udf(
230        message: String,
231        collector: Option<PromqlAnnotationCollector>,
232    ) -> ScalarUDF {
233        ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
234            Self::float_null_name(),
235            AnnotationReturn::FloatNull,
236            AnnotationLevel::Info,
237            message,
238            collector,
239        ))
240    }
241
242    pub fn bool_false_udf(
243        message: String,
244        collector: Option<PromqlAnnotationCollector>,
245    ) -> ScalarUDF {
246        ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
247            Self::bool_false_name(),
248            AnnotationReturn::BooleanFalse,
249            AnnotationLevel::Info,
250            message,
251            collector,
252        ))
253    }
254
255    pub fn bool_true_udf(
256        message: String,
257        collector: Option<PromqlAnnotationCollector>,
258    ) -> ScalarUDF {
259        ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
260            Self::bool_true_name(),
261            AnnotationReturn::BooleanTrue,
262            AnnotationLevel::Info,
263            message,
264            collector,
265        ))
266    }
267
268    pub fn warning_bool_false_udf(
269        message: String,
270        collector: Option<PromqlAnnotationCollector>,
271    ) -> ScalarUDF {
272        ScalarUDF::new_from_impl(NativeHistogramAnnotationUdf::new(
273            Self::bool_false_name(),
274            AnnotationReturn::BooleanFalse,
275            AnnotationLevel::Warning,
276            message,
277            collector,
278        ))
279    }
280}
281
282fn record_info(collector: &Option<PromqlAnnotationCollector>, message: impl Into<String>) {
283    if let Some(collector) = collector {
284        collector.record_info(message);
285    }
286}
287
288fn record_warning(collector: &Option<PromqlAnnotationCollector>, message: impl Into<String>) {
289    if let Some(collector) = collector {
290        collector.record_warning(message);
291    }
292}
293
294fn record_custom_reconciliation(
295    collector: &Option<PromqlAnnotationCollector>,
296    name: &'static str,
297    lhs: &NativeHistogram,
298    rhs: &NativeHistogram,
299) {
300    if lhs.needs_custom_reconciliation(rhs) {
301        record_info(
302            collector,
303            format!("{name}: reconciled native histograms with different custom buckets"),
304        );
305    }
306}
307
308fn record_counter_reset_contradiction(
309    collector: &Option<PromqlAnnotationCollector>,
310    name: &'static str,
311    lhs: &NativeHistogram,
312    rhs: &NativeHistogram,
313) {
314    if lhs.counter_reset_hints_contradict(rhs) {
315        record_counter_reset_contradiction_warning(collector, name);
316    }
317}
318
319fn record_counter_reset_contradiction_warning(
320    collector: &Option<PromqlAnnotationCollector>,
321    name: &'static str,
322) {
323    record_warning(
324        collector,
325        format!("{name}: native histogram counter reset hints contradict"),
326    );
327}
328
329fn scalar_histogram_udf(
330    name: &'static str,
331    extra_input_types: Vec<DataType>,
332    calc: fn(&NativeHistogram, &[ColumnarValue], usize, usize, &'static str) -> DfResult<f64>,
333) -> ScalarUDF {
334    let mut input_types = vec![native_histogram_arrow_type()];
335    input_types.extend(extra_input_types);
336    create_udf(
337        name,
338        input_types,
339        DataType::Float64,
340        Volatility::Volatile,
341        Arc::new(move |input: &[ColumnarValue]| {
342            if input.is_empty() {
343                return Err(DataFusionError::Plan(format!(
344                    "{name} requires a native histogram argument"
345                )));
346            }
347            let histograms = extract_histogram_array(&input[0], name)?;
348            let histograms = histograms
349                .as_any()
350                .downcast_ref::<StructArray>()
351                .expect("validated native histogram struct");
352            let mut result = Float64Builder::with_capacity(histograms.len());
353            for row in 0..histograms.len() {
354                match read_histogram(histograms, row)? {
355                    Some(histogram) => {
356                        result.append_value(calc(&histogram, input, row, histograms.len(), name)?)
357                    }
358                    None => result.append_null(),
359                }
360            }
361            Ok(ColumnarValue::Array(Arc::new(result.finish())))
362        }) as _,
363    )
364}
365
366fn histogram_pair_udf(
367    name: &'static str,
368    op: fn(&NativeHistogram, &NativeHistogram) -> Option<NativeHistogram>,
369) -> ScalarUDF {
370    histogram_pair_udf_with_collector(name, op, None)
371}
372
373fn histogram_pair_udf_with_collector(
374    name: &'static str,
375    op: fn(&NativeHistogram, &NativeHistogram) -> Option<NativeHistogram>,
376    collector: Option<PromqlAnnotationCollector>,
377) -> ScalarUDF {
378    create_udf(
379        name,
380        vec![native_histogram_arrow_type(), native_histogram_arrow_type()],
381        native_histogram_arrow_type(),
382        Volatility::Volatile,
383        Arc::new(move |input: &[ColumnarValue]| {
384            let lhs = extract_histogram_array(&input[0], name)?;
385            let rhs = extract_histogram_array(&input[1], name)?;
386            if lhs.len() != rhs.len() {
387                return Err(DataFusionError::Execution(format!(
388                    "{name}: native histogram argument length mismatch: {} vs {}",
389                    lhs.len(),
390                    rhs.len()
391                )));
392            }
393
394            let lhs = lhs
395                .as_any()
396                .downcast_ref::<StructArray>()
397                .expect("validated native histogram struct");
398            let rhs = rhs
399                .as_any()
400                .downcast_ref::<StructArray>()
401                .expect("validated native histogram struct");
402            let mut result = Vec::with_capacity(lhs.len());
403            for row in 0..lhs.len() {
404                result.push(
405                    match (read_histogram(lhs, row)?, read_histogram(rhs, row)?) {
406                        (Some(lhs), Some(rhs)) => {
407                            record_custom_reconciliation(&collector, name, &lhs, &rhs);
408                            record_counter_reset_contradiction(&collector, name, &lhs, &rhs);
409                            let result = op(&lhs, &rhs);
410                            if result.is_none() {
411                                record_warning(
412                                    &collector,
413                                    format!(
414                                    "{name}: dropped native histogram sample with incompatible schemas"
415                                    ),
416                                );
417                            }
418                            result
419                        }
420                        _ => None,
421                    },
422                );
423            }
424            Ok(ColumnarValue::Array(build_histogram_array(&result)))
425        }) as _,
426    )
427}
428
429fn histogram_transform_udf(
430    name: &'static str,
431    op: fn(NativeHistogram) -> NativeHistogram,
432) -> ScalarUDF {
433    create_udf(
434        name,
435        vec![native_histogram_arrow_type()],
436        native_histogram_arrow_type(),
437        Volatility::Volatile,
438        Arc::new(move |input: &[ColumnarValue]| {
439            let histograms = extract_histogram_array(&input[0], name)?;
440            let histograms = histograms
441                .as_any()
442                .downcast_ref::<StructArray>()
443                .expect("validated native histogram struct");
444            let mut result = Vec::with_capacity(histograms.len());
445            for row in 0..histograms.len() {
446                result.push(read_histogram(histograms, row)?.map(op));
447            }
448            Ok(ColumnarValue::Array(build_histogram_array(&result)))
449        }) as _,
450    )
451}
452
453fn histogram_string_udf(name: &'static str) -> ScalarUDF {
454    create_udf(
455        name,
456        vec![native_histogram_arrow_type()],
457        DataType::Utf8,
458        Volatility::Volatile,
459        Arc::new(move |input: &[ColumnarValue]| {
460            let histograms = extract_histogram_array(&input[0], name)?;
461            let histograms = histograms
462                .as_any()
463                .downcast_ref::<StructArray>()
464                .expect("validated native histogram struct");
465            let mut result = StringBuilder::with_capacity(histograms.len(), histograms.len() * 32);
466            for row in 0..histograms.len() {
467                match read_histogram(histograms, row)? {
468                    Some(histogram) => result.append_value(histogram.promql_string()),
469                    None => result.append_null(),
470                }
471            }
472            Ok(ColumnarValue::Array(Arc::new(result.finish())))
473        }) as _,
474    )
475}
476
477fn histogram_scalar_udf(
478    name: &'static str,
479    input_types: Vec<DataType>,
480    histogram_index: usize,
481    scalar_index: usize,
482    op: fn(NativeHistogram, f64) -> Option<NativeHistogram>,
483) -> ScalarUDF {
484    create_udf(
485        name,
486        input_types,
487        native_histogram_arrow_type(),
488        Volatility::Volatile,
489        Arc::new(move |input: &[ColumnarValue]| {
490            let histograms = extract_histogram_array(&input[histogram_index], name)?;
491            let histograms = histograms
492                .as_any()
493                .downcast_ref::<StructArray>()
494                .expect("validated native histogram struct");
495            let mut result = Vec::with_capacity(histograms.len());
496            for row in 0..histograms.len() {
497                result.push(match read_histogram(histograms, row)? {
498                    Some(histogram) => {
499                        let scalar =
500                            read_scalar_f64_arg(&input[scalar_index], row, histograms.len(), name)?;
501                        op(histogram, scalar)
502                    }
503                    None => None,
504                });
505            }
506            Ok(ColumnarValue::Array(build_histogram_array(&result)))
507        }) as _,
508    )
509}
510
511fn histogram_compare_udf(
512    name: &'static str,
513    op: fn(&NativeHistogram, &NativeHistogram) -> bool,
514) -> ScalarUDF {
515    create_udf(
516        name,
517        vec![native_histogram_arrow_type(), native_histogram_arrow_type()],
518        DataType::Boolean,
519        Volatility::Volatile,
520        Arc::new(move |input: &[ColumnarValue]| {
521            let lhs = extract_histogram_array(&input[0], name)?;
522            let rhs = extract_histogram_array(&input[1], name)?;
523            if lhs.len() != rhs.len() {
524                return Err(DataFusionError::Execution(format!(
525                    "{name}: native histogram argument length mismatch: {} vs {}",
526                    lhs.len(),
527                    rhs.len()
528                )));
529            }
530
531            let lhs = lhs
532                .as_any()
533                .downcast_ref::<StructArray>()
534                .expect("validated native histogram struct");
535            let rhs = rhs
536                .as_any()
537                .downcast_ref::<StructArray>()
538                .expect("validated native histogram struct");
539            let mut result = Vec::with_capacity(lhs.len());
540            for row in 0..lhs.len() {
541                result.push(
542                    match (read_histogram(lhs, row)?, read_histogram(rhs, row)?) {
543                        (Some(lhs), Some(rhs)) => Some(op(&lhs, &rhs)),
544                        _ => None,
545                    },
546                );
547            }
548            Ok(ColumnarValue::Array(Arc::new(BooleanArray::from(result))))
549        }) as _,
550    )
551}
552
553pub struct NativeHistogramAdd;
554
555impl NativeHistogramAdd {
556    pub const fn name() -> &'static str {
557        "prom_native_histogram_add"
558    }
559
560    pub fn scalar_udf() -> ScalarUDF {
561        histogram_pair_udf(Self::name(), NativeHistogram::add)
562    }
563
564    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
565        histogram_pair_udf_with_collector(Self::name(), NativeHistogram::add, collector)
566    }
567}
568
569pub struct NativeHistogramSub;
570
571impl NativeHistogramSub {
572    pub const fn name() -> &'static str {
573        "prom_native_histogram_sub"
574    }
575
576    pub fn scalar_udf() -> ScalarUDF {
577        histogram_pair_udf(Self::name(), NativeHistogram::sub)
578    }
579
580    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
581        histogram_pair_udf_with_collector(Self::name(), NativeHistogram::sub, collector)
582    }
583}
584
585pub struct NativeHistogramMulScalar;
586
587impl NativeHistogramMulScalar {
588    pub const fn name() -> &'static str {
589        "prom_native_histogram_mul_scalar"
590    }
591
592    pub fn scalar_udf() -> ScalarUDF {
593        histogram_scalar_udf(
594            Self::name(),
595            vec![native_histogram_arrow_type(), DataType::Float64],
596            0,
597            1,
598            |histogram, scalar| Some(histogram.scale(scalar)),
599        )
600    }
601}
602
603pub struct NativeHistogramScalarMul;
604
605impl NativeHistogramScalarMul {
606    pub const fn name() -> &'static str {
607        "prom_native_histogram_scalar_mul"
608    }
609
610    pub fn scalar_udf() -> ScalarUDF {
611        histogram_scalar_udf(
612            Self::name(),
613            vec![DataType::Float64, native_histogram_arrow_type()],
614            1,
615            0,
616            |histogram, scalar| Some(histogram.scale(scalar)),
617        )
618    }
619}
620
621pub struct NativeHistogramDivScalar;
622
623impl NativeHistogramDivScalar {
624    pub const fn name() -> &'static str {
625        "prom_native_histogram_div_scalar"
626    }
627
628    pub fn scalar_udf() -> ScalarUDF {
629        histogram_scalar_udf(
630            Self::name(),
631            vec![native_histogram_arrow_type(), DataType::Float64],
632            0,
633            1,
634            |histogram, scalar| Some(histogram.divide_by(scalar)),
635        )
636    }
637}
638
639pub struct NativeHistogramNeg;
640
641impl NativeHistogramNeg {
642    pub const fn name() -> &'static str {
643        "prom_native_histogram_neg"
644    }
645
646    pub fn scalar_udf() -> ScalarUDF {
647        histogram_transform_udf(Self::name(), NativeHistogram::negated)
648    }
649}
650
651pub struct NativeHistogramEq;
652
653impl NativeHistogramEq {
654    pub const fn name() -> &'static str {
655        "prom_native_histogram_eq"
656    }
657
658    pub fn scalar_udf() -> ScalarUDF {
659        histogram_compare_udf(Self::name(), NativeHistogram::promql_eq)
660    }
661}
662
663pub struct NativeHistogramNotEq;
664
665impl NativeHistogramNotEq {
666    pub const fn name() -> &'static str {
667        "prom_native_histogram_not_eq"
668    }
669
670    pub fn scalar_udf() -> ScalarUDF {
671        histogram_compare_udf(Self::name(), |lhs, rhs| !lhs.promql_eq(rhs))
672    }
673}
674
675pub struct NativeHistogramCount;
676
677impl NativeHistogramCount {
678    pub const fn name() -> &'static str {
679        "prom_native_histogram_count"
680    }
681
682    pub fn scalar_udf() -> ScalarUDF {
683        scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
684            Ok(histogram.count)
685        })
686    }
687}
688
689pub struct NativeHistogramSum;
690
691impl NativeHistogramSum {
692    pub const fn name() -> &'static str {
693        "prom_native_histogram_sum"
694    }
695
696    pub fn scalar_udf() -> ScalarUDF {
697        scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
698            Ok(histogram.sum)
699        })
700    }
701}
702
703pub struct NativeHistogramAvg;
704
705impl NativeHistogramAvg {
706    pub const fn name() -> &'static str {
707        "prom_native_histogram_avg"
708    }
709
710    pub fn scalar_udf() -> ScalarUDF {
711        scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
712            Ok(histogram.sum / histogram.count)
713        })
714    }
715}
716
717pub struct NativeHistogramStddev;
718
719impl NativeHistogramStddev {
720    pub const fn name() -> &'static str {
721        "prom_native_histogram_stddev"
722    }
723
724    pub fn scalar_udf() -> ScalarUDF {
725        scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
726            Ok(histogram.estimated_stddev())
727        })
728    }
729}
730
731pub struct NativeHistogramStdvar;
732
733impl NativeHistogramStdvar {
734    pub const fn name() -> &'static str {
735        "prom_native_histogram_stdvar"
736    }
737
738    pub fn scalar_udf() -> ScalarUDF {
739        scalar_histogram_udf(Self::name(), vec![], |histogram, _, _, _, _| {
740            Ok(histogram.estimated_stdvar())
741        })
742    }
743}
744
745/// Formats float samples as PromQL label values.
746pub struct PromqlFloatToString;
747
748impl PromqlFloatToString {
749    pub const fn name() -> &'static str {
750        "prom_float_to_string"
751    }
752
753    pub fn scalar_udf() -> ScalarUDF {
754        create_udf(
755            Self::name(),
756            vec![DataType::Float64],
757            DataType::Utf8,
758            Volatility::Volatile,
759            Arc::new(|input: &[ColumnarValue]| {
760                let values = extract_array(&input[0])?;
761                let values = values
762                    .as_any()
763                    .downcast_ref::<Float64Array>()
764                    .expect("validated Float64 input");
765                let mut result = StringBuilder::new();
766                for value in values.iter() {
767                    match value {
768                        Some(value) => result.append_value(format_prometheus_float(value)),
769                        None => result.append_null(),
770                    }
771                }
772                Ok(ColumnarValue::Array(Arc::new(result.finish())))
773            }),
774        )
775    }
776}
777
778pub struct NativeHistogramToString;
779
780impl NativeHistogramToString {
781    pub const fn name() -> &'static str {
782        "prom_native_histogram_to_string"
783    }
784
785    pub fn scalar_udf() -> ScalarUDF {
786        histogram_string_udf(Self::name())
787    }
788}
789
790pub struct NativeHistogramQuantile;
791
792impl NativeHistogramQuantile {
793    pub const fn name() -> &'static str {
794        "prom_native_histogram_quantile"
795    }
796
797    pub fn scalar_udf() -> ScalarUDF {
798        scalar_histogram_udf(
799            Self::name(),
800            vec![DataType::Float64],
801            |histogram, input, row, len, name| {
802                Ok(histogram.quantile(read_scalar_f64_arg(&input[1], row, len, name)?))
803            },
804        )
805    }
806}
807
808pub struct NativeHistogramFraction;
809
810impl NativeHistogramFraction {
811    pub const fn name() -> &'static str {
812        "prom_native_histogram_fraction"
813    }
814
815    pub fn scalar_udf() -> ScalarUDF {
816        scalar_histogram_udf(
817            Self::name(),
818            vec![DataType::Float64, DataType::Float64],
819            |histogram, input, row, len, name| {
820                let lower = read_scalar_f64_arg(&input[1], row, len, name)?;
821                let upper = read_scalar_f64_arg(&input[2], row, len, name)?;
822                Ok(histogram.fraction(lower, upper))
823            },
824        )
825    }
826}
827
828#[derive(Debug, Clone, Copy)]
829enum NativeHistogramAggregateKind {
830    Sum,
831    Avg,
832}
833
834impl NativeHistogramAggregateKind {
835    const fn name(self) -> &'static str {
836        match self {
837            Self::Sum => NativeHistogramAggSum::name(),
838            Self::Avg => NativeHistogramAggAvg::name(),
839        }
840    }
841
842    const fn needs_count(self) -> bool {
843        matches!(self, Self::Avg)
844    }
845}
846
847pub struct NativeHistogramAggSum;
848
849impl NativeHistogramAggSum {
850    pub const fn name() -> &'static str {
851        "prom_native_histogram_agg_sum"
852    }
853
854    pub fn aggregate_udf() -> AggregateUDF {
855        native_histogram_aggregate_udf(NativeHistogramAggregateKind::Sum, None)
856    }
857
858    pub fn aggregate_udf_with_collector(
859        collector: Option<PromqlAnnotationCollector>,
860    ) -> AggregateUDF {
861        native_histogram_aggregate_udf(NativeHistogramAggregateKind::Sum, collector)
862    }
863}
864
865pub struct NativeHistogramAggAvg;
866
867impl NativeHistogramAggAvg {
868    pub const fn name() -> &'static str {
869        "prom_native_histogram_agg_avg"
870    }
871
872    pub fn aggregate_udf() -> AggregateUDF {
873        native_histogram_aggregate_udf(NativeHistogramAggregateKind::Avg, None)
874    }
875
876    pub fn aggregate_udf_with_collector(
877        collector: Option<PromqlAnnotationCollector>,
878    ) -> AggregateUDF {
879        native_histogram_aggregate_udf(NativeHistogramAggregateKind::Avg, collector)
880    }
881}
882
883#[derive(Debug)]
884struct NativeHistogramAggregateAccumulator {
885    kind: NativeHistogramAggregateKind,
886    value: Option<NativeHistogram>,
887    count: u64,
888    dropped_incompatible: bool,
889    counter_reset_seen: bool,
890    not_counter_reset_seen: bool,
891    collector: Option<PromqlAnnotationCollector>,
892}
893
894impl NativeHistogramAggregateAccumulator {
895    fn new(
896        kind: NativeHistogramAggregateKind,
897        collector: Option<PromqlAnnotationCollector>,
898    ) -> Self {
899        Self {
900            kind,
901            value: None,
902            count: 0,
903            dropped_incompatible: false,
904            counter_reset_seen: false,
905            not_counter_reset_seen: false,
906            collector,
907        }
908    }
909
910    fn from_args(
911        kind: NativeHistogramAggregateKind,
912        collector: Option<PromqlAnnotationCollector>,
913        _args: AccumulatorArgs,
914    ) -> DfResult<Box<dyn DfAccumulator>> {
915        Ok(Box::new(Self::new(kind, collector)))
916    }
917
918    fn observe_reset_hints(&mut self, counter_reset_seen: bool, not_counter_reset_seen: bool) {
919        self.counter_reset_seen |= counter_reset_seen;
920        self.not_counter_reset_seen |= not_counter_reset_seen;
921        if self.counter_reset_seen && self.not_counter_reset_seen {
922            record_counter_reset_contradiction_warning(&self.collector, self.kind.name());
923        }
924    }
925
926    fn push_histogram(&mut self, histogram: NativeHistogram, count: u64) -> DfResult<()> {
927        if self.kind.needs_count() && count == 0 {
928            return Ok(());
929        }
930
931        self.observe_reset_hints(
932            histogram.reset_hint == COUNTER_RESET_HINT,
933            histogram.reset_hint == NOT_COUNTER_RESET_HINT,
934        );
935        if self.dropped_incompatible {
936            return Ok(());
937        }
938        let combined_count = if self.kind.needs_count() {
939            self.count.checked_add(count).ok_or_else(|| {
940                DataFusionError::Execution(format!(
941                    "{}: native histogram sample count overflow",
942                    self.kind.name()
943                ))
944            })?
945        } else {
946            self.count
947        };
948        let value = match self.value.take() {
949            Some(value) => {
950                record_custom_reconciliation(&self.collector, self.kind.name(), &value, &histogram);
951                let combined = match self.kind {
952                    NativeHistogramAggregateKind::Sum => value.add(&histogram),
953                    NativeHistogramAggregateKind::Avg => {
954                        weighted_histogram_mean(value, self.count, histogram, count, combined_count)
955                    }
956                };
957                match combined {
958                    Some(value) => Some(value),
959                    None => {
960                        self.record_incompatible();
961                        None
962                    }
963                }
964            }
965            None => Some(histogram),
966        };
967        if !self.dropped_incompatible {
968            self.value = value;
969            self.count = combined_count;
970        }
971        Ok(())
972    }
973
974    fn mark_incompatible(&mut self) {
975        self.value = None;
976        self.count = 0;
977        self.dropped_incompatible = true;
978    }
979
980    fn record_incompatible(&mut self) {
981        self.mark_incompatible();
982        record_warning(
983            &self.collector,
984            format!(
985                "{}: dropped native histogram aggregate with incompatible schemas",
986                self.kind.name()
987            ),
988        );
989    }
990}
991
992fn weighted_histogram_mean(
993    left: NativeHistogram,
994    left_count: u64,
995    right: NativeHistogram,
996    right_count: u64,
997    total_count: u64,
998) -> Option<NativeHistogram> {
999    let total_count = total_count as f64;
1000    left.scale(left_count as f64 / total_count)
1001        .add(&right.scale(right_count as f64 / total_count))
1002}
1003
1004fn range_fold_histograms(
1005    samples: Vec<NativeHistogram>,
1006    kind: NativeHistogramAggregateKind,
1007    name: &'static str,
1008    collector: &Option<PromqlAnnotationCollector>,
1009) -> Option<NativeHistogram> {
1010    if samples
1011        .iter()
1012        .any(|histogram| histogram.reset_hint == COUNTER_RESET_HINT)
1013        && samples
1014            .iter()
1015            .any(|histogram| histogram.reset_hint == NOT_COUNTER_RESET_HINT)
1016    {
1017        record_counter_reset_contradiction_warning(collector, name);
1018    }
1019
1020    let mut value = None;
1021    let mut count = 0u64;
1022    for histogram in samples {
1023        value = match value {
1024            Some(value) => {
1025                record_custom_reconciliation(collector, name, &value, &histogram);
1026                let next_count = count.checked_add(1)?;
1027                let combined = match kind {
1028                    NativeHistogramAggregateKind::Sum => value.add(&histogram),
1029                    NativeHistogramAggregateKind::Avg => {
1030                        weighted_histogram_mean(value, count, histogram, 1, next_count)
1031                    }
1032                };
1033                match combined {
1034                    Some(value) => Some(value),
1035                    None => {
1036                        record_warning(
1037                            collector,
1038                            format!(
1039                                "{name}: dropped native histogram range with incompatible schemas"
1040                            ),
1041                        );
1042                        return None;
1043                    }
1044                }
1045            }
1046            None => Some(histogram),
1047        };
1048        count = count.checked_add(1)?;
1049    }
1050    value
1051}
1052
1053#[derive(Debug, Clone, Copy)]
1054enum NativeHistogramRangeHistogramKind {
1055    Sum,
1056    Avg,
1057    Last,
1058}
1059
1060#[derive(Debug, Clone, Copy)]
1061enum NativeHistogramRangeFloatKind {
1062    Absent,
1063    Count,
1064    Present,
1065    Changes,
1066    Resets,
1067}
1068
1069fn collect_window_histograms(
1070    histograms: &StructArray,
1071    offset: usize,
1072    length: usize,
1073) -> DfResult<Option<Vec<NativeHistogram>>> {
1074    let mut samples = Vec::with_capacity(length);
1075    for row in offset..offset + length {
1076        let Some(histogram) = read_histogram(histograms, row)? else {
1077            return Ok(None);
1078        };
1079        samples.push(histogram);
1080    }
1081    Ok(Some(samples))
1082}
1083
1084fn native_histogram_range_histogram(
1085    input: &[ColumnarValue],
1086    kind: NativeHistogramRangeHistogramKind,
1087    func_name: &'static str,
1088    collector: Option<PromqlAnnotationCollector>,
1089) -> DfResult<ColumnarValue> {
1090    if input.len() != 2 {
1091        return Err(DataFusionError::Plan(format!(
1092            "{func_name} function should have 2 inputs"
1093        )));
1094    }
1095
1096    let ts_range = extract_range_dict(
1097        &input[0],
1098        func_name,
1099        "timestamp range vector",
1100        &DataType::Timestamp(TimeUnit::Millisecond, None),
1101    )?;
1102    let value_range = extract_range_dict(
1103        &input[1],
1104        func_name,
1105        "value range vector",
1106        &native_histogram_arrow_type(),
1107    )?;
1108    if ts_range.keys().values() != value_range.keys().values() {
1109        return Err(DataFusionError::Execution(format!(
1110            "{func_name}: timestamp and value ranges should have the same window layout"
1111        )));
1112    }
1113
1114    let histograms = value_range
1115        .values()
1116        .as_any()
1117        .downcast_ref::<StructArray>()
1118        .expect("validated native histogram range");
1119    let mut result = Vec::with_capacity(value_range.keys().len());
1120    for key in value_range.keys().values() {
1121        let (offset, length) = unpack(*key);
1122        let offset = offset as usize;
1123        let length = length as usize;
1124        if length == 0 {
1125            result.push(None);
1126            continue;
1127        }
1128        if matches!(kind, NativeHistogramRangeHistogramKind::Last) {
1129            let histogram = if (offset..offset + length).any(|row| histograms.is_null(row)) {
1130                None
1131            } else {
1132                read_histogram(histograms, offset + length - 1)?
1133            };
1134            result.push(histogram);
1135            continue;
1136        }
1137        let Some(samples) = collect_window_histograms(histograms, offset, length)? else {
1138            result.push(None);
1139            continue;
1140        };
1141        let histogram = match kind {
1142            NativeHistogramRangeHistogramKind::Sum => range_fold_histograms(
1143                samples,
1144                NativeHistogramAggregateKind::Sum,
1145                func_name,
1146                &collector,
1147            ),
1148            NativeHistogramRangeHistogramKind::Avg => range_fold_histograms(
1149                samples,
1150                NativeHistogramAggregateKind::Avg,
1151                func_name,
1152                &collector,
1153            ),
1154            NativeHistogramRangeHistogramKind::Last => samples.last().cloned(),
1155        };
1156        result.push(histogram);
1157    }
1158
1159    Ok(ColumnarValue::Array(build_histogram_array(&result)))
1160}
1161
1162fn native_histogram_range_float(
1163    input: &[ColumnarValue],
1164    kind: NativeHistogramRangeFloatKind,
1165    func_name: &'static str,
1166) -> DfResult<ColumnarValue> {
1167    if input.len() != 2 {
1168        return Err(DataFusionError::Plan(format!(
1169            "{func_name} function should have 2 inputs"
1170        )));
1171    }
1172
1173    let ts_range = extract_range_dict(
1174        &input[0],
1175        func_name,
1176        "timestamp range vector",
1177        &DataType::Timestamp(TimeUnit::Millisecond, None),
1178    )?;
1179    let value_range = extract_range_dict(
1180        &input[1],
1181        func_name,
1182        "value range vector",
1183        &native_histogram_arrow_type(),
1184    )?;
1185    if ts_range.keys().values() != value_range.keys().values() {
1186        return Err(DataFusionError::Execution(format!(
1187            "{func_name}: timestamp and value ranges should have the same window layout"
1188        )));
1189    }
1190
1191    let timestamps = ts_range
1192        .values()
1193        .as_any()
1194        .downcast_ref::<TimestampMillisecondArray>()
1195        .expect("validated timestamp range")
1196        .values();
1197    let histograms = value_range
1198        .values()
1199        .as_any()
1200        .downcast_ref::<StructArray>()
1201        .expect("validated native histogram range");
1202    let mut result = Float64Builder::with_capacity(value_range.keys().len());
1203    for key in value_range.keys().values() {
1204        let (offset, length) = unpack(*key);
1205        let offset = offset as usize;
1206        let length = length as usize;
1207        if length == 0 {
1208            match kind {
1209                NativeHistogramRangeFloatKind::Absent => result.append_value(1.0),
1210                _ => result.append_null(),
1211            }
1212            continue;
1213        }
1214        if matches!(kind, NativeHistogramRangeFloatKind::Absent) {
1215            result.append_null();
1216            continue;
1217        }
1218        if matches!(
1219            kind,
1220            NativeHistogramRangeFloatKind::Count | NativeHistogramRangeFloatKind::Present
1221        ) {
1222            if (offset..offset + length).any(|row| histograms.is_null(row)) {
1223                result.append_null();
1224            } else if matches!(kind, NativeHistogramRangeFloatKind::Count) {
1225                result.append_value(length as f64);
1226            } else {
1227                result.append_value(1.0);
1228            }
1229            continue;
1230        }
1231        let Some(samples) = collect_window_histograms(histograms, offset, length)? else {
1232            result.append_null();
1233            continue;
1234        };
1235        let value = match kind {
1236            NativeHistogramRangeFloatKind::Absent => {
1237                result.append_null();
1238                continue;
1239            }
1240            NativeHistogramRangeFloatKind::Count => length as f64,
1241            NativeHistogramRangeFloatKind::Present => 1.0,
1242            NativeHistogramRangeFloatKind::Changes => samples
1243                .windows(2)
1244                .filter(|pair| !pair[0].promql_eq(&pair[1]))
1245                .count() as f64,
1246            NativeHistogramRangeFloatKind::Resets => samples
1247                .windows(2)
1248                .zip(timestamps[offset..offset + length].windows(2))
1249                .filter(|(pair, ts_pair)| {
1250                    (pair[0].reset_hint == GAUGE_RESET_HINT)
1251                        != (pair[1].reset_hint == GAUGE_RESET_HINT)
1252                        || pair[1].detect_counter_reset(&pair[0], ts_pair[0], ts_pair[1])
1253                })
1254                .count() as f64,
1255        };
1256        result.append_value(value);
1257    }
1258
1259    Ok(ColumnarValue::Array(Arc::new(result.finish())))
1260}
1261
1262fn create_native_range_histogram_udf(
1263    name: &'static str,
1264    kind: NativeHistogramRangeHistogramKind,
1265    collector: Option<PromqlAnnotationCollector>,
1266) -> ScalarUDF {
1267    create_udf(
1268        name,
1269        vec![
1270            RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
1271            RangeArray::convert_data_type(native_histogram_arrow_type()),
1272        ],
1273        native_histogram_arrow_type(),
1274        Volatility::Volatile,
1275        Arc::new(move |input: &[ColumnarValue]| {
1276            native_histogram_range_histogram(input, kind, name, collector.clone())
1277        }) as _,
1278    )
1279}
1280
1281fn create_native_range_float_udf(
1282    name: &'static str,
1283    kind: NativeHistogramRangeFloatKind,
1284) -> ScalarUDF {
1285    create_udf(
1286        name,
1287        vec![
1288            RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
1289            RangeArray::convert_data_type(native_histogram_arrow_type()),
1290        ],
1291        DataType::Float64,
1292        Volatility::Volatile,
1293        Arc::new(move |input: &[ColumnarValue]| native_histogram_range_float(input, kind, name))
1294            as _,
1295    )
1296}
1297
1298pub struct NativeHistogramSumOverTime;
1299pub struct NativeHistogramAvgOverTime;
1300pub struct NativeHistogramAbsentOverTime;
1301pub struct NativeHistogramCountOverTime;
1302pub struct NativeHistogramLastOverTime;
1303pub struct NativeHistogramPresentOverTime;
1304pub struct NativeHistogramChanges;
1305pub struct NativeHistogramResets;
1306
1307impl NativeHistogramSumOverTime {
1308    pub const fn name() -> &'static str {
1309        "prom_native_histogram_sum_over_time"
1310    }
1311
1312    pub fn scalar_udf() -> ScalarUDF {
1313        Self::scalar_udf_with_collector(None)
1314    }
1315
1316    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1317        create_native_range_histogram_udf(
1318            Self::name(),
1319            NativeHistogramRangeHistogramKind::Sum,
1320            collector,
1321        )
1322    }
1323}
1324
1325impl NativeHistogramAvgOverTime {
1326    pub const fn name() -> &'static str {
1327        "prom_native_histogram_avg_over_time"
1328    }
1329
1330    pub fn scalar_udf() -> ScalarUDF {
1331        Self::scalar_udf_with_collector(None)
1332    }
1333
1334    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1335        create_native_range_histogram_udf(
1336            Self::name(),
1337            NativeHistogramRangeHistogramKind::Avg,
1338            collector,
1339        )
1340    }
1341}
1342
1343impl NativeHistogramAbsentOverTime {
1344    pub const fn name() -> &'static str {
1345        "prom_native_histogram_absent_over_time"
1346    }
1347
1348    pub fn scalar_udf() -> ScalarUDF {
1349        create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Absent)
1350    }
1351}
1352
1353impl NativeHistogramCountOverTime {
1354    pub const fn name() -> &'static str {
1355        "prom_native_histogram_count_over_time"
1356    }
1357
1358    pub fn scalar_udf() -> ScalarUDF {
1359        create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Count)
1360    }
1361}
1362
1363impl NativeHistogramLastOverTime {
1364    pub const fn name() -> &'static str {
1365        "prom_native_histogram_last_over_time"
1366    }
1367
1368    pub fn scalar_udf() -> ScalarUDF {
1369        create_native_range_histogram_udf(
1370            Self::name(),
1371            NativeHistogramRangeHistogramKind::Last,
1372            None,
1373        )
1374    }
1375}
1376
1377impl NativeHistogramPresentOverTime {
1378    pub const fn name() -> &'static str {
1379        "prom_native_histogram_present_over_time"
1380    }
1381
1382    pub fn scalar_udf() -> ScalarUDF {
1383        create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Present)
1384    }
1385}
1386
1387impl NativeHistogramChanges {
1388    pub const fn name() -> &'static str {
1389        "prom_native_histogram_changes"
1390    }
1391
1392    pub fn scalar_udf() -> ScalarUDF {
1393        create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Changes)
1394    }
1395}
1396
1397impl NativeHistogramResets {
1398    pub const fn name() -> &'static str {
1399        "prom_native_histogram_resets"
1400    }
1401
1402    pub fn scalar_udf() -> ScalarUDF {
1403        create_native_range_float_udf(Self::name(), NativeHistogramRangeFloatKind::Resets)
1404    }
1405}
1406
1407/// Coordinated float/native-histogram range evaluation.
1408///
1409/// The function name is passed as the first scalar argument so these two UDF names are enough for
1410/// distributed plan decoding. The remaining leading arguments are timestamp, float, and histogram
1411/// ranges, followed by the ordinary function arguments.
1412pub struct MixedRange;
1413
1414impl MixedRange {
1415    const fn float_name() -> &'static str {
1416        "prom_mixed_range_float"
1417    }
1418
1419    const fn histogram_name() -> &'static str {
1420        "prom_mixed_range_histogram"
1421    }
1422
1423    pub fn float_udf(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1424        ScalarUDF::new_from_impl(MixedRangeUdf::new(MixedRangeOutput::Float, collector))
1425    }
1426
1427    pub fn histogram_udf(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
1428        ScalarUDF::new_from_impl(MixedRangeUdf::new(MixedRangeOutput::Histogram, collector))
1429    }
1430}
1431
1432#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1433enum MixedRangeOutput {
1434    Float,
1435    Histogram,
1436}
1437
1438impl MixedRangeOutput {
1439    fn name(self) -> &'static str {
1440        match self {
1441            Self::Float => MixedRange::float_name(),
1442            Self::Histogram => MixedRange::histogram_name(),
1443        }
1444    }
1445
1446    fn data_type(self) -> DataType {
1447        match self {
1448            Self::Float => DataType::Float64,
1449            Self::Histogram => native_histogram_arrow_type(),
1450        }
1451    }
1452}
1453
1454#[derive(Debug, Clone)]
1455struct MixedRangeUdf {
1456    output: MixedRangeOutput,
1457    signature: Signature,
1458    collector: Option<PromqlAnnotationCollector>,
1459}
1460
1461impl MixedRangeUdf {
1462    fn new(output: MixedRangeOutput, collector: Option<PromqlAnnotationCollector>) -> Self {
1463        Self {
1464            output,
1465            signature: Signature::variadic_any(Volatility::Volatile),
1466            collector,
1467        }
1468    }
1469}
1470
1471impl PartialEq for MixedRangeUdf {
1472    fn eq(&self, other: &Self) -> bool {
1473        self.output == other.output
1474    }
1475}
1476
1477impl Eq for MixedRangeUdf {}
1478
1479impl Hash for MixedRangeUdf {
1480    fn hash<H: Hasher>(&self, state: &mut H) {
1481        self.output.hash(state);
1482    }
1483}
1484
1485impl ScalarUDFImpl for MixedRangeUdf {
1486    fn as_any(&self) -> &dyn Any {
1487        self
1488    }
1489
1490    fn name(&self) -> &str {
1491        self.output.name()
1492    }
1493
1494    fn signature(&self) -> &Signature {
1495        &self.signature
1496    }
1497
1498    fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
1499        Ok(self.output.data_type())
1500    }
1501
1502    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult<ColumnarValue> {
1503        let collector = args
1504            .config_options
1505            .extensions
1506            .get::<PromqlAnnotationCollector>()
1507            .cloned()
1508            .or_else(|| self.collector.clone());
1509        mixed_range(&args, self.output, collector)
1510    }
1511}
1512
1513#[derive(Debug, Clone, Copy)]
1514enum MixedRangeFunction {
1515    Rate,
1516    Increase,
1517    Delta,
1518    IDelta,
1519    IRate,
1520    Changes,
1521    Resets,
1522    AvgOverTime,
1523    MinOverTime,
1524    MaxOverTime,
1525    SumOverTime,
1526    CountOverTime,
1527    LastOverTime,
1528    AbsentOverTime,
1529    PresentOverTime,
1530    StddevOverTime,
1531    StdvarOverTime,
1532    QuantileOverTime,
1533    Deriv,
1534    PredictLinear,
1535    DoubleExponentialSmoothing,
1536    HoltWinters,
1537}
1538
1539impl MixedRangeFunction {
1540    fn parse(value: &ColumnarValue) -> DfResult<Self> {
1541        let name = match value {
1542            ColumnarValue::Scalar(ScalarValue::Utf8(Some(name)))
1543            | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(name)))
1544            | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(name))) => name.as_str(),
1545            other => {
1546                return Err(DataFusionError::Execution(format!(
1547                    "mixed range function name must be a non-null string scalar, found {}",
1548                    other.data_type()
1549                )));
1550            }
1551        };
1552
1553        match name {
1554            "rate" => Ok(Self::Rate),
1555            "increase" => Ok(Self::Increase),
1556            "delta" => Ok(Self::Delta),
1557            "idelta" => Ok(Self::IDelta),
1558            "irate" => Ok(Self::IRate),
1559            "changes" => Ok(Self::Changes),
1560            "resets" => Ok(Self::Resets),
1561            "avg_over_time" => Ok(Self::AvgOverTime),
1562            "min_over_time" => Ok(Self::MinOverTime),
1563            "max_over_time" => Ok(Self::MaxOverTime),
1564            "sum_over_time" => Ok(Self::SumOverTime),
1565            "count_over_time" => Ok(Self::CountOverTime),
1566            "last_over_time" => Ok(Self::LastOverTime),
1567            "absent_over_time" => Ok(Self::AbsentOverTime),
1568            "present_over_time" => Ok(Self::PresentOverTime),
1569            "stddev_over_time" => Ok(Self::StddevOverTime),
1570            "stdvar_over_time" => Ok(Self::StdvarOverTime),
1571            "quantile_over_time" => Ok(Self::QuantileOverTime),
1572            "deriv" => Ok(Self::Deriv),
1573            "predict_linear" => Ok(Self::PredictLinear),
1574            "double_exponential_smoothing" => Ok(Self::DoubleExponentialSmoothing),
1575            "holt_winters" => Ok(Self::HoltWinters),
1576            _ => Err(DataFusionError::Execution(format!(
1577                "unsupported mixed range function: {name}"
1578            ))),
1579        }
1580    }
1581
1582    fn name(self) -> &'static str {
1583        match self {
1584            Self::Rate => "rate",
1585            Self::Increase => "increase",
1586            Self::Delta => "delta",
1587            Self::IDelta => "idelta",
1588            Self::IRate => "irate",
1589            Self::Changes => "changes",
1590            Self::Resets => "resets",
1591            Self::AvgOverTime => "avg_over_time",
1592            Self::MinOverTime => "min_over_time",
1593            Self::MaxOverTime => "max_over_time",
1594            Self::SumOverTime => "sum_over_time",
1595            Self::CountOverTime => "count_over_time",
1596            Self::LastOverTime => "last_over_time",
1597            Self::AbsentOverTime => "absent_over_time",
1598            Self::PresentOverTime => "present_over_time",
1599            Self::StddevOverTime => "stddev_over_time",
1600            Self::StdvarOverTime => "stdvar_over_time",
1601            Self::QuantileOverTime => "quantile_over_time",
1602            Self::Deriv => "deriv",
1603            Self::PredictLinear => "predict_linear",
1604            Self::DoubleExponentialSmoothing => "double_exponential_smoothing",
1605            Self::HoltWinters => "holt_winters",
1606        }
1607    }
1608
1609    fn policy(self) -> MixedRangePolicy {
1610        match self {
1611            Self::Rate | Self::Increase | Self::Delta | Self::AvgOverTime | Self::SumOverTime => {
1612                MixedRangePolicy::DropMixed
1613            }
1614            Self::IDelta | Self::IRate => MixedRangePolicy::LastTwo,
1615            Self::LastOverTime => MixedRangePolicy::Last,
1616            Self::Changes
1617            | Self::Resets
1618            | Self::CountOverTime
1619            | Self::AbsentOverTime
1620            | Self::PresentOverTime => MixedRangePolicy::Combined,
1621            Self::MinOverTime
1622            | Self::MaxOverTime
1623            | Self::StddevOverTime
1624            | Self::StdvarOverTime
1625            | Self::QuantileOverTime
1626            | Self::Deriv
1627            | Self::PredictLinear
1628            | Self::DoubleExponentialSmoothing
1629            | Self::HoltWinters => MixedRangePolicy::FloatOnly,
1630        }
1631    }
1632
1633    fn float_udf(self) -> Option<ScalarUDF> {
1634        match self {
1635            Self::Rate => Some(Rate::scalar_udf()),
1636            Self::Increase => Some(Increase::scalar_udf()),
1637            Self::Delta => Some(crate::functions::Delta::scalar_udf()),
1638            Self::IDelta => Some(IDelta::<false>::scalar_udf()),
1639            Self::IRate => Some(IDelta::<true>::scalar_udf()),
1640            Self::AvgOverTime => Some(AvgOverTime::scalar_udf()),
1641            Self::MinOverTime => Some(MinOverTime::scalar_udf()),
1642            Self::MaxOverTime => Some(MaxOverTime::scalar_udf()),
1643            Self::SumOverTime => Some(SumOverTime::scalar_udf()),
1644            Self::LastOverTime => Some(LastOverTime::scalar_udf()),
1645            Self::StddevOverTime => Some(StddevOverTime::scalar_udf()),
1646            Self::StdvarOverTime => Some(StdvarOverTime::scalar_udf()),
1647            Self::QuantileOverTime => Some(QuantileOverTime::scalar_udf()),
1648            Self::Deriv => Some(Deriv::scalar_udf()),
1649            Self::PredictLinear => Some(PredictLinear::scalar_udf()),
1650            Self::DoubleExponentialSmoothing | Self::HoltWinters => {
1651                Some(DoubleExponentialSmoothing::scalar_udf())
1652            }
1653            Self::Changes
1654            | Self::Resets
1655            | Self::CountOverTime
1656            | Self::AbsentOverTime
1657            | Self::PresentOverTime => None,
1658        }
1659    }
1660
1661    fn histogram_udf(self, collector: Option<PromqlAnnotationCollector>) -> Option<ScalarUDF> {
1662        match self {
1663            Self::Rate => Some(NativeHistogramRate::scalar_udf_with_collector(collector)),
1664            Self::Increase => Some(NativeHistogramIncrease::scalar_udf_with_collector(
1665                collector,
1666            )),
1667            Self::Delta => Some(NativeHistogramDelta::scalar_udf_with_collector(collector)),
1668            Self::IDelta => Some(NativeHistogramIDelta::scalar_udf_with_collector(collector)),
1669            Self::IRate => Some(NativeHistogramIRate::scalar_udf_with_collector(collector)),
1670            Self::AvgOverTime => Some(NativeHistogramAvgOverTime::scalar_udf_with_collector(
1671                collector,
1672            )),
1673            Self::SumOverTime => Some(NativeHistogramSumOverTime::scalar_udf_with_collector(
1674                collector,
1675            )),
1676            Self::LastOverTime => Some(NativeHistogramLastOverTime::scalar_udf()),
1677            _ => None,
1678        }
1679    }
1680}
1681
1682#[derive(Debug, Clone, Copy)]
1683enum MixedRangePolicy {
1684    DropMixed,
1685    LastTwo,
1686    Last,
1687    Combined,
1688    FloatOnly,
1689}
1690
1691#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1692enum SampleLane {
1693    Float,
1694    Histogram,
1695}
1696
1697fn mixed_range(
1698    args: &ScalarFunctionArgs,
1699    output: MixedRangeOutput,
1700    collector: Option<PromqlAnnotationCollector>,
1701) -> DfResult<ColumnarValue> {
1702    if args.args.len() < 4 {
1703        return Err(DataFusionError::Plan(format!(
1704            "{} function should have at least 4 inputs",
1705            output.name()
1706        )));
1707    }
1708    let function = MixedRangeFunction::parse(&args.args[0])?;
1709    let name = function.name();
1710    let ts_range = extract_range_dict(
1711        &args.args[1],
1712        name,
1713        "timestamp range vector",
1714        &DataType::Timestamp(TimeUnit::Millisecond, None),
1715    )?;
1716    let float_range = extract_range_dict(
1717        &args.args[2],
1718        name,
1719        "float range vector",
1720        &DataType::Float64,
1721    )?;
1722    let histogram_range = extract_range_dict(
1723        &args.args[3],
1724        name,
1725        "native histogram range vector",
1726        &native_histogram_arrow_type(),
1727    )?;
1728    let keys = ts_range.keys().values();
1729    if float_range.keys().values() != keys || histogram_range.keys().values() != keys {
1730        return Err(DataFusionError::Execution(format!(
1731            "{name}: timestamp, float, and native histogram ranges should have the same layout"
1732        )));
1733    }
1734    if args.number_rows != keys.len() {
1735        return Err(DataFusionError::Execution(format!(
1736            "{name}: range inputs have {} windows but the batch has {} rows",
1737            keys.len(),
1738            args.number_rows
1739        )));
1740    }
1741
1742    let timestamps = ts_range
1743        .values()
1744        .as_any()
1745        .downcast_ref::<TimestampMillisecondArray>()
1746        .expect("validated timestamp range");
1747    let floats = float_range
1748        .values()
1749        .as_any()
1750        .downcast_ref::<Float64Array>()
1751        .expect("validated float range");
1752    let histograms = histogram_range
1753        .values()
1754        .as_any()
1755        .downcast_ref::<StructArray>()
1756        .expect("validated native histogram range");
1757    if floats.len() != timestamps.len() || histograms.len() != timestamps.len() {
1758        return Err(DataFusionError::Execution(format!(
1759            "{name}: timestamp, float, and native histogram values should be row-aligned"
1760        )));
1761    }
1762
1763    let bounds = checked_window_bounds(keys, timestamps.len(), name)?;
1764    let float_valid = (0..floats.len())
1765        .map(|row| floats.is_valid(row))
1766        .collect::<Vec<_>>();
1767    let histogram_valid = (0..histograms.len())
1768        .map(|row| histograms.is_valid(row))
1769        .collect::<Vec<_>>();
1770    let float_prefix = validity_prefix(&float_valid, name)?;
1771    let histogram_prefix = validity_prefix(&histogram_valid, name)?;
1772
1773    if matches!(function.policy(), MixedRangePolicy::Combined) {
1774        if output != MixedRangeOutput::Float {
1775            return Err(DataFusionError::Execution(format!(
1776                "{name} does not return native histograms"
1777            )));
1778        }
1779        return combined_range_float(
1780            function,
1781            &bounds,
1782            timestamps,
1783            floats,
1784            histograms,
1785            &float_valid,
1786            &histogram_valid,
1787            &float_prefix,
1788            &histogram_prefix,
1789        );
1790    }
1791
1792    let selections = select_mixed_range_lanes(
1793        function,
1794        &bounds,
1795        timestamps,
1796        &float_valid,
1797        &histogram_valid,
1798        &float_prefix,
1799        &histogram_prefix,
1800        &collector,
1801    );
1802    let (lane, values, valid, prefix) = match output {
1803        MixedRangeOutput::Float => (
1804            SampleLane::Float,
1805            floats as &dyn Array,
1806            float_valid.as_slice(),
1807            float_prefix.as_slice(),
1808        ),
1809        MixedRangeOutput::Histogram => (
1810            SampleLane::Histogram,
1811            histograms as &dyn Array,
1812            histogram_valid.as_slice(),
1813            histogram_prefix.as_slice(),
1814        ),
1815    };
1816    let input = compact_lane_input(
1817        timestamps,
1818        values,
1819        valid,
1820        prefix,
1821        &bounds,
1822        &selections,
1823        lane,
1824        name,
1825    )?;
1826
1827    let udf = match output {
1828        MixedRangeOutput::Float => function.float_udf(),
1829        MixedRangeOutput::Histogram => function.histogram_udf(collector),
1830    }
1831    .ok_or_else(|| {
1832        DataFusionError::Execution(format!(
1833            "{name} does not return {} values",
1834            match output {
1835                MixedRangeOutput::Float => "float",
1836                MixedRangeOutput::Histogram => "native histogram",
1837            }
1838        ))
1839    })?;
1840    let mut input = input;
1841    input.extend_from_slice(&args.args[4..]);
1842    invoke_range_udf(udf, input, args, bounds.len())
1843}
1844
1845/// Decodes packed range keys into validated half-open `(offset, end)` bounds.
1846fn checked_window_bounds(
1847    keys: &[i64],
1848    value_len: usize,
1849    name: &str,
1850) -> DfResult<Vec<(usize, usize)>> {
1851    keys.iter()
1852        .map(|key| {
1853            let (offset, length) = unpack(*key);
1854            let offset = offset as usize;
1855            let end = offset
1856                .checked_add(length as usize)
1857                .filter(|end| *end <= value_len)
1858                .ok_or_else(|| {
1859                    DataFusionError::Execution(format!(
1860                        "{name}: invalid range ({offset}, {length}) for {value_len} values"
1861                    ))
1862                })?;
1863            Ok((offset, end))
1864        })
1865        .collect()
1866}
1867
1868/// Builds prefix counts where `prefix[i]` is the number of valid samples in `valid[..i]`.
1869fn validity_prefix(valid: &[bool], name: &str) -> DfResult<Vec<usize>> {
1870    let capacity = valid.len().checked_add(1).ok_or_else(|| {
1871        DataFusionError::Execution(format!("{name}: sample validity length overflow"))
1872    })?;
1873    let mut prefix = Vec::with_capacity(capacity);
1874    prefix.push(0usize);
1875    for is_valid in valid {
1876        prefix.push(
1877            prefix
1878                .last()
1879                .copied()
1880                .unwrap()
1881                .checked_add(usize::from(*is_valid))
1882                .ok_or_else(|| {
1883                    DataFusionError::Execution(format!("{name}: sample count overflow"))
1884                })?,
1885        );
1886    }
1887    Ok(prefix)
1888}
1889
1890/// Returns the number of valid samples in the half-open window `[offset, end)`.
1891fn valid_count(prefix: &[usize], offset: usize, end: usize) -> usize {
1892    prefix[end]
1893        .checked_sub(prefix[offset])
1894        .expect("validity prefix is monotonic")
1895}
1896
1897/// Selects the sample lane for each window and records policy-required annotations.
1898#[allow(clippy::too_many_arguments)]
1899fn select_mixed_range_lanes(
1900    function: MixedRangeFunction,
1901    bounds: &[(usize, usize)],
1902    timestamps: &TimestampMillisecondArray,
1903    float_valid: &[bool],
1904    histogram_valid: &[bool],
1905    float_prefix: &[usize],
1906    histogram_prefix: &[usize],
1907    collector: &Option<PromqlAnnotationCollector>,
1908) -> Vec<Option<SampleLane>> {
1909    bounds
1910        .iter()
1911        .map(|(offset, end)| {
1912            let float_count = valid_count(float_prefix, *offset, *end);
1913            let histogram_count = valid_count(histogram_prefix, *offset, *end);
1914            match function.policy() {
1915                MixedRangePolicy::DropMixed => match (float_count > 0, histogram_count > 0) {
1916                    (true, true) => {
1917                        record_warning(
1918                            collector,
1919                            format!(
1920                                "{}: encountered a mix of float and native histogram samples",
1921                                function.name()
1922                            ),
1923                        );
1924                        None
1925                    }
1926                    (true, false) => Some(SampleLane::Float),
1927                    (false, true) => Some(SampleLane::Histogram),
1928                    (false, false) => None,
1929                },
1930                MixedRangePolicy::FloatOnly => {
1931                    if float_count > 0 && histogram_count > 0 {
1932                        record_info(
1933                            collector,
1934                            format!(
1935                                "{}: ignored native histogram samples",
1936                                function.name()
1937                            ),
1938                        );
1939                    }
1940                    (float_count > 0).then_some(SampleLane::Float)
1941                }
1942                MixedRangePolicy::Last => (*offset..*end).rev().find_map(|row| {
1943                    if histogram_valid[row] {
1944                        Some(SampleLane::Histogram)
1945                    } else if float_valid[row] {
1946                        Some(SampleLane::Float)
1947                    } else {
1948                        None
1949                    }
1950                }),
1951                MixedRangePolicy::LastTwo => {
1952                    let mut last_two = Vec::with_capacity(2);
1953                    for row in (*offset..*end).rev() {
1954                        // Prometheus keeps a float as the newest sample if both lanes have the
1955                        // same timestamp, while the histogram becomes the preceding sample.
1956                        if float_valid[row] {
1957                            last_two.push((SampleLane::Float, timestamps.value(row)));
1958                        }
1959                        if last_two.len() < 2 && histogram_valid[row] {
1960                            last_two.push((SampleLane::Histogram, timestamps.value(row)));
1961                        }
1962                        if last_two.len() == 2 {
1963                            break;
1964                        }
1965                    }
1966                    if last_two.len() < 2 || last_two[0].1 == last_two[1].1 {
1967                        None
1968                    } else if last_two[0].0 == last_two[1].0 {
1969                        Some(last_two[0].0)
1970                    } else {
1971                        record_warning(
1972                            collector,
1973                            format!(
1974                                "{}: encountered a mix of float and native histogram samples in the last two points",
1975                                function.name()
1976                            ),
1977                        );
1978                        None
1979                    }
1980                }
1981                MixedRangePolicy::Combined => unreachable!(),
1982            }
1983        })
1984        .collect()
1985}
1986
1987/// Filters null placeholders from one sample lane and remaps its selected windows.
1988/// Windows assigned to the other lane become empty.
1989#[allow(clippy::too_many_arguments)]
1990fn compact_lane_input(
1991    timestamps: &TimestampMillisecondArray,
1992    values: &dyn Array,
1993    valid: &[bool],
1994    prefix: &[usize],
1995    bounds: &[(usize, usize)],
1996    selections: &[Option<SampleLane>],
1997    lane: SampleLane,
1998    name: &str,
1999) -> DfResult<Vec<ColumnarValue>> {
2000    let mask = BooleanArray::from(valid.to_vec());
2001    let filtered_timestamps = filter(timestamps, &mask)?;
2002    let filtered_values = filter(values, &mask)?;
2003    let ranges = bounds
2004        .iter()
2005        .zip(selections)
2006        .map(|((offset, end), selected)| {
2007            let compact_offset = prefix[*offset];
2008            let compact_length = if *selected == Some(lane) {
2009                valid_count(prefix, *offset, *end)
2010            } else {
2011                0
2012            };
2013            Ok((
2014                u32::try_from(compact_offset).map_err(|_| {
2015                    DataFusionError::Execution(format!(
2016                        "{name}: compacted range offset exceeds u32"
2017                    ))
2018                })?,
2019                u32::try_from(compact_length).map_err(|_| {
2020                    DataFusionError::Execution(format!(
2021                        "{name}: compacted range length exceeds u32"
2022                    ))
2023                })?,
2024            ))
2025        })
2026        .collect::<DfResult<Vec<_>>>()?;
2027    let timestamp_range = RangeArray::from_ranges(filtered_timestamps, ranges.clone())
2028        .map_err(DataFusionError::from)?;
2029    let value_range =
2030        RangeArray::from_ranges(filtered_values, ranges).map_err(DataFusionError::from)?;
2031    Ok(vec![
2032        ColumnarValue::Array(Arc::new(timestamp_range.into_dict())),
2033        ColumnarValue::Array(Arc::new(value_range.into_dict())),
2034    ])
2035}
2036
2037fn invoke_range_udf(
2038    udf: ScalarUDF,
2039    input: Vec<ColumnarValue>,
2040    outer_args: &ScalarFunctionArgs,
2041    number_rows: usize,
2042) -> DfResult<ColumnarValue> {
2043    let arg_fields = input
2044        .iter()
2045        .enumerate()
2046        .map(|(index, value)| Arc::new(Field::new(format!("arg_{index}"), value.data_type(), true)))
2047        .collect();
2048    udf.invoke_with_args(ScalarFunctionArgs {
2049        args: input,
2050        arg_fields,
2051        number_rows,
2052        return_field: outer_args.return_field.clone(),
2053        config_options: outer_args.config_options.clone(),
2054    })
2055}
2056
2057enum MixedSample {
2058    Float(f64),
2059    Histogram(NativeHistogram),
2060}
2061
2062#[allow(clippy::too_many_arguments)]
2063fn combined_range_float(
2064    function: MixedRangeFunction,
2065    bounds: &[(usize, usize)],
2066    timestamps: &TimestampMillisecondArray,
2067    floats: &Float64Array,
2068    histograms: &StructArray,
2069    float_valid: &[bool],
2070    histogram_valid: &[bool],
2071    float_prefix: &[usize],
2072    histogram_prefix: &[usize],
2073) -> DfResult<ColumnarValue> {
2074    let mut result = Float64Builder::with_capacity(bounds.len());
2075    for (offset, end) in bounds {
2076        let sample_count = valid_count(float_prefix, *offset, *end)
2077            .checked_add(valid_count(histogram_prefix, *offset, *end))
2078            .ok_or_else(|| {
2079                DataFusionError::Execution(format!("{}: sample count overflow", function.name()))
2080            })?;
2081        match function {
2082            MixedRangeFunction::CountOverTime => {
2083                if sample_count == 0 {
2084                    result.append_null();
2085                } else {
2086                    result.append_value(sample_count as f64);
2087                }
2088            }
2089            MixedRangeFunction::AbsentOverTime => {
2090                if sample_count == 0 {
2091                    result.append_value(1.0);
2092                } else {
2093                    result.append_null();
2094                }
2095            }
2096            MixedRangeFunction::PresentOverTime => {
2097                if sample_count == 0 {
2098                    result.append_null();
2099                } else {
2100                    result.append_value(1.0);
2101                }
2102            }
2103            MixedRangeFunction::Changes | MixedRangeFunction::Resets => {
2104                if sample_count == 0 {
2105                    result.append_null();
2106                    continue;
2107                }
2108                let mut count = 0usize;
2109                let mut previous = None;
2110                for row in *offset..*end {
2111                    if float_valid[row] {
2112                        count += mixed_transition(
2113                            function,
2114                            &mut previous,
2115                            timestamps.value(row),
2116                            MixedSample::Float(floats.value(row)),
2117                        );
2118                    }
2119                    if histogram_valid[row] {
2120                        let histogram = read_histogram(histograms, row)?
2121                            .expect("validated native histogram sample");
2122                        count += mixed_transition(
2123                            function,
2124                            &mut previous,
2125                            timestamps.value(row),
2126                            MixedSample::Histogram(histogram),
2127                        );
2128                    }
2129                }
2130                result.append_value(count as f64);
2131            }
2132            _ => {
2133                return Err(DataFusionError::Internal(format!(
2134                    "{} does not support combined range evaluation",
2135                    function.name()
2136                )));
2137            }
2138        }
2139    }
2140    Ok(ColumnarValue::Array(Arc::new(result.finish())))
2141}
2142
2143fn mixed_transition(
2144    function: MixedRangeFunction,
2145    previous: &mut Option<(i64, MixedSample)>,
2146    timestamp: i64,
2147    current: MixedSample,
2148) -> usize {
2149    let changed = previous.as_ref().is_some_and(|(previous_ts, previous)| {
2150        match (function, previous, &current) {
2151            (
2152                MixedRangeFunction::Changes,
2153                MixedSample::Float(previous),
2154                MixedSample::Float(current),
2155            ) => current != previous && !(current.is_nan() && previous.is_nan()),
2156            (
2157                MixedRangeFunction::Changes,
2158                MixedSample::Histogram(previous),
2159                MixedSample::Histogram(current),
2160            ) => !current.promql_eq(previous),
2161            (MixedRangeFunction::Changes, _, _) => true,
2162            (
2163                MixedRangeFunction::Resets,
2164                MixedSample::Float(previous),
2165                MixedSample::Float(current),
2166            ) => current < previous,
2167            (
2168                MixedRangeFunction::Resets,
2169                MixedSample::Histogram(previous),
2170                MixedSample::Histogram(current),
2171            ) => {
2172                (previous.reset_hint == GAUGE_RESET_HINT)
2173                    != (current.reset_hint == GAUGE_RESET_HINT)
2174                    || current.detect_counter_reset(previous, *previous_ts, timestamp)
2175            }
2176            (MixedRangeFunction::Resets, _, _) => true,
2177            _ => unreachable!(),
2178        }
2179    });
2180    *previous = Some((timestamp, current));
2181    usize::from(changed)
2182}
2183
2184fn native_histogram_scalar(histogram: Option<NativeHistogram>) -> ScalarValue {
2185    let array = build_histogram_array(&[histogram]);
2186    let histogram = array
2187        .as_any()
2188        .downcast_ref::<StructArray>()
2189        .expect("native histogram array is a StructArray")
2190        .clone();
2191    ScalarValue::Struct(Arc::new(histogram))
2192}
2193
2194fn native_histogram_aggregate_udf(
2195    kind: NativeHistogramAggregateKind,
2196    collector: Option<PromqlAnnotationCollector>,
2197) -> AggregateUDF {
2198    let state_types = if kind.needs_count() {
2199        vec![
2200            native_histogram_arrow_type(),
2201            DataType::UInt64,
2202            DataType::Boolean,
2203            DataType::Boolean,
2204            DataType::Boolean,
2205        ]
2206    } else {
2207        vec![
2208            native_histogram_arrow_type(),
2209            DataType::Boolean,
2210            DataType::Boolean,
2211            DataType::Boolean,
2212        ]
2213    };
2214
2215    create_udaf(
2216        kind.name(),
2217        vec![native_histogram_arrow_type()],
2218        Arc::new(native_histogram_arrow_type()),
2219        Volatility::Volatile,
2220        Arc::new(move |args| {
2221            NativeHistogramAggregateAccumulator::from_args(kind, collector.clone(), args)
2222        }),
2223        Arc::new(state_types),
2224    )
2225}
2226
2227impl DfAccumulator for NativeHistogramAggregateAccumulator {
2228    fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
2229        let histograms = values
2230            .first()
2231            .and_then(|array| array.as_any().downcast_ref::<StructArray>())
2232            .ok_or_else(|| {
2233                DataFusionError::Execution(format!(
2234                    "{}: expected native histogram struct input",
2235                    self.kind.name()
2236                ))
2237            })?;
2238
2239        for row in 0..histograms.len() {
2240            let Some(histogram) = read_histogram(histograms, row)? else {
2241                continue;
2242            };
2243            self.push_histogram(histogram, 1)?;
2244        }
2245
2246        Ok(())
2247    }
2248
2249    fn evaluate(&mut self) -> DfResult<ScalarValue> {
2250        let histogram = match (self.kind, self.dropped_incompatible, self.value.clone()) {
2251            (_, true, _) => None,
2252            (_, false, None) => None,
2253            (NativeHistogramAggregateKind::Sum, false, value) => value,
2254            (NativeHistogramAggregateKind::Avg, false, Some(value)) if self.count > 0 => {
2255                Some(value)
2256            }
2257            (NativeHistogramAggregateKind::Avg, _, _) => None,
2258        };
2259
2260        Ok(native_histogram_scalar(histogram))
2261    }
2262
2263    fn size(&self) -> usize {
2264        size_of::<Self>()
2265            + self.value.as_ref().map_or(0, |histogram| {
2266                histogram.custom_values.capacity() * size_of::<f64>()
2267                    + histogram.positive_spans.capacity() * size_of::<Span>()
2268                    + histogram.negative_spans.capacity() * size_of::<Span>()
2269                    + histogram.positive_buckets.capacity() * size_of::<f64>()
2270                    + histogram.negative_buckets.capacity() * size_of::<f64>()
2271            })
2272    }
2273
2274    fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
2275        let mut state = vec![native_histogram_scalar(self.value.clone())];
2276        if self.kind.needs_count() {
2277            state.push(ScalarValue::UInt64(Some(self.count)));
2278        }
2279        state.push(ScalarValue::Boolean(Some(self.dropped_incompatible)));
2280        state.push(ScalarValue::Boolean(Some(self.counter_reset_seen)));
2281        state.push(ScalarValue::Boolean(Some(self.not_counter_reset_seen)));
2282        Ok(state)
2283    }
2284
2285    fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> {
2286        if states.is_empty() {
2287            return Ok(());
2288        }
2289
2290        let histograms = states[0]
2291            .as_any()
2292            .downcast_ref::<StructArray>()
2293            .ok_or_else(|| {
2294                DataFusionError::Execution(format!(
2295                    "{}: expected native histogram struct state",
2296                    self.kind.name()
2297                ))
2298            })?;
2299        let counts = if self.kind.needs_count() {
2300            Some(
2301                states
2302                    .get(1)
2303                    .and_then(|array| array.as_any().downcast_ref::<UInt64Array>())
2304                    .ok_or_else(|| {
2305                        DataFusionError::Execution(format!(
2306                            "{}: expected UInt64 count state",
2307                            self.kind.name()
2308                        ))
2309                    })?,
2310            )
2311        } else {
2312            None
2313        };
2314        let dropped_index = if self.kind.needs_count() { 2 } else { 1 };
2315        let dropped = states
2316            .get(dropped_index)
2317            .and_then(|array| array.as_any().downcast_ref::<BooleanArray>())
2318            .ok_or_else(|| {
2319                DataFusionError::Execution(format!(
2320                    "{}: expected Boolean dropped state",
2321                    self.kind.name()
2322                ))
2323            })?;
2324        let counter_reset_seen = states
2325            .get(dropped_index + 1)
2326            .and_then(|array| array.as_any().downcast_ref::<BooleanArray>())
2327            .ok_or_else(|| {
2328                DataFusionError::Execution(format!(
2329                    "{}: expected Boolean counter reset state",
2330                    self.kind.name()
2331                ))
2332            })?;
2333        let not_counter_reset_seen = states
2334            .get(dropped_index + 2)
2335            .and_then(|array| array.as_any().downcast_ref::<BooleanArray>())
2336            .ok_or_else(|| {
2337                DataFusionError::Execution(format!(
2338                    "{}: expected Boolean not-counter-reset state",
2339                    self.kind.name()
2340                ))
2341            })?;
2342
2343        for row in 0..histograms.len() {
2344            self.observe_reset_hints(
2345                counter_reset_seen.value(row),
2346                not_counter_reset_seen.value(row),
2347            );
2348            if dropped.value(row) {
2349                self.mark_incompatible();
2350            }
2351            if self.dropped_incompatible {
2352                continue;
2353            }
2354            let Some(histogram) = read_histogram(histograms, row)? else {
2355                continue;
2356            };
2357            let count = counts.map(|counts| counts.value(row)).unwrap_or(1);
2358            self.push_histogram(histogram, count)?;
2359        }
2360
2361        Ok(())
2362    }
2363}
2364
2365fn histogram_delta(
2366    samples: &[NativeHistogram],
2367    timestamps: &[i64],
2368    is_counter: bool,
2369) -> Option<NativeHistogram> {
2370    if samples.len() < 2 || samples.len() != timestamps.len() {
2371        return None;
2372    }
2373
2374    if !is_counter {
2375        return samples
2376            .last()?
2377            .sub(samples.first()?)
2378            .map(NativeHistogram::into_gauge);
2379    }
2380
2381    let first_reset = samples[1].detect_counter_reset(&samples[0], timestamps[0], timestamps[1]);
2382    let (initial, reset_scan_start) = if first_reset {
2383        // The first sample is irrelevant after an immediate reset. Adopt the
2384        // second sample's layout so an incompatible pre-reset layout is ignored.
2385        (samples[1].zero_like(), 2)
2386    } else {
2387        (samples[0].clone(), 1)
2388    };
2389    let mut result = samples.last()?.sub(&initial)?;
2390    for index in reset_scan_start..samples.len() {
2391        if samples[index].detect_counter_reset(
2392            &samples[index - 1],
2393            timestamps[index - 1],
2394            timestamps[index],
2395        ) {
2396            result = result.add(&samples[index - 1])?;
2397        }
2398    }
2399    Some(result.into_gauge())
2400}
2401
2402fn idelta_value(
2403    samples: &[NativeHistogram],
2404    is_rate: bool,
2405    previous_ts: i64,
2406    current_ts: i64,
2407    sampled_interval_secs: f64,
2408) -> Option<NativeHistogram> {
2409    if samples.len() < 2 {
2410        return None;
2411    }
2412    let previous = &samples[samples.len() - 2];
2413    let current = samples.last()?;
2414    let result = if is_rate && current.detect_counter_reset(previous, previous_ts, current_ts) {
2415        current.clone()
2416    } else {
2417        current.sub(previous)?
2418    };
2419    Some(
2420        if is_rate {
2421            result.scale(1.0 / sampled_interval_secs)
2422        } else {
2423            result
2424        }
2425        .into_gauge(),
2426    )
2427}
2428
2429fn native_extrapolated_rate<const IS_COUNTER: bool, const IS_RATE: bool>(
2430    input: &[ColumnarValue],
2431    range_length: i64,
2432    func_name: &'static str,
2433    collector: Option<PromqlAnnotationCollector>,
2434) -> DfResult<ColumnarValue> {
2435    if input.len() != 4 {
2436        return Err(DataFusionError::Plan(format!(
2437            "{func_name} function should have 4 inputs"
2438        )));
2439    }
2440
2441    let ts_dict = extract_range_dict(
2442        &input[0],
2443        func_name,
2444        "timestamp range vector",
2445        &DataType::Timestamp(TimeUnit::Millisecond, None),
2446    )?;
2447    let value_dict = extract_range_dict(
2448        &input[1],
2449        func_name,
2450        "value range vector",
2451        &native_histogram_arrow_type(),
2452    )?;
2453    let eval_ts = extract_array(&input[2])?;
2454    let eval_ts = eval_ts
2455        .as_any()
2456        .downcast_ref::<TimestampMillisecondArray>()
2457        .ok_or_else(|| {
2458            DataFusionError::Execution(format!(
2459                "{func_name}: expect evaluation timestamp vector as Timestamp(Millisecond), found {}",
2460                eval_ts.data_type()
2461            ))
2462        })?;
2463
2464    let keys = ts_dict.keys().values();
2465    if value_dict.keys().values() != keys || eval_ts.len() != keys.len() {
2466        return Err(DataFusionError::Execution(format!(
2467            "{func_name}: timestamp, value, and evaluation ranges should have the same layout"
2468        )));
2469    }
2470
2471    let all_timestamps = ts_dict
2472        .values()
2473        .as_any()
2474        .downcast_ref::<TimestampMillisecondArray>()
2475        .expect("validated timestamp range")
2476        .values();
2477    let all_histograms = value_dict
2478        .values()
2479        .as_any()
2480        .downcast_ref::<StructArray>()
2481        .expect("validated native histogram range");
2482    let range_length_secs = range_length as f64 / 1000.0;
2483    let mut result = Vec::with_capacity(keys.len());
2484
2485    for index in 0..keys.len() {
2486        let (raw_offset, raw_length) = unpack(keys[index]);
2487        let offset = raw_offset as usize;
2488        let length = raw_length as usize;
2489        if length == 0 {
2490            result.push(None);
2491            continue;
2492        }
2493
2494        let mut samples = Vec::with_capacity(length);
2495        let mut has_null = false;
2496        for row in offset..offset + length {
2497            let Some(histogram) = read_histogram(all_histograms, row)? else {
2498                has_null = true;
2499                break;
2500            };
2501            samples.push(histogram);
2502        }
2503        if has_null {
2504            result.push(None);
2505            continue;
2506        }
2507
2508        let first_ts = all_timestamps[offset];
2509        let last_ts = all_timestamps[offset + length - 1];
2510        let range_end = eval_ts.value(index);
2511        let range_start = range_end - range_length;
2512        let synthetic_zero_timestamp = IS_COUNTER
2513            .then_some(samples[0].start_timestamp)
2514            .flatten()
2515            .filter(|start| *start != 0 && range_start < *start && *start < first_ts);
2516        let synthetic_zero_start = synthetic_zero_timestamp.is_some();
2517        if length < 2 && !synthetic_zero_start {
2518            result.push(None);
2519            continue;
2520        }
2521
2522        let wrong_flavor = if IS_COUNTER {
2523            samples
2524                .iter()
2525                .any(|histogram| histogram.reset_hint == GAUGE_RESET_HINT)
2526        } else {
2527            samples[0].reset_hint != GAUGE_RESET_HINT
2528                || samples[samples.len() - 1].reset_hint != GAUGE_RESET_HINT
2529        };
2530        if wrong_flavor {
2531            let expected = if IS_COUNTER { "counter" } else { "gauge" };
2532            record_warning(
2533                &collector,
2534                format!("{func_name}: native histogram input should be a {expected} histogram"),
2535            );
2536        }
2537
2538        let timestamps = &all_timestamps[offset..offset + length];
2539        for pair in samples.windows(2) {
2540            record_custom_reconciliation(&collector, func_name, &pair[0], &pair[1]);
2541        }
2542        let mut histogram = if length == 1 {
2543            samples[0].clone()
2544        } else if let Some(histogram) = histogram_delta(&samples, timestamps, IS_COUNTER) {
2545            histogram
2546        } else {
2547            record_warning(
2548                &collector,
2549                format!("{func_name}: dropped native histogram range with incompatible schemas"),
2550            );
2551            result.push(None);
2552            continue;
2553        };
2554        if synthetic_zero_start && length > 1 {
2555            let Some(with_synthetic_zero) = histogram.add(&samples[0]) else {
2556                record_warning(
2557                    &collector,
2558                    format!(
2559                        "{func_name}: dropped native histogram range with incompatible schemas"
2560                    ),
2561                );
2562                result.push(None);
2563                continue;
2564            };
2565            histogram = with_synthetic_zero;
2566        }
2567
2568        let real_sampled_interval_ms = (last_ts - first_ts) as f64;
2569        let sampled_interval_ms = synthetic_zero_timestamp
2570            .map(|start| (last_ts - start) as f64)
2571            .unwrap_or(real_sampled_interval_ms);
2572        if sampled_interval_ms <= 0.0 {
2573            result.push(None);
2574            continue;
2575        }
2576        let average_interval_ms = if length > 1 {
2577            real_sampled_interval_ms / (length - 1) as f64
2578        } else {
2579            0.0
2580        };
2581        let mut duration_to_start_ms = if synthetic_zero_start {
2582            0.0
2583        } else {
2584            (first_ts - range_start) as f64
2585        };
2586        let duration_to_end_ms = (range_end - last_ts) as f64;
2587
2588        if IS_COUNTER && !synthetic_zero_start && histogram.count > 0.0 && samples[0].count >= 0.0 {
2589            let duration_to_zero = sampled_interval_ms * (samples[0].count / histogram.count);
2590            if duration_to_zero < duration_to_start_ms {
2591                duration_to_start_ms = duration_to_zero;
2592            }
2593        }
2594
2595        let extrapolation_threshold = average_interval_ms * 1.1;
2596        let mut extrapolated_interval_ms = sampled_interval_ms;
2597        if duration_to_start_ms < extrapolation_threshold {
2598            extrapolated_interval_ms += duration_to_start_ms;
2599        } else {
2600            extrapolated_interval_ms += average_interval_ms / 2.0;
2601        }
2602        if duration_to_end_ms < extrapolation_threshold {
2603            extrapolated_interval_ms += duration_to_end_ms;
2604        } else {
2605            extrapolated_interval_ms += average_interval_ms / 2.0;
2606        }
2607
2608        let mut factor = extrapolated_interval_ms / sampled_interval_ms;
2609        if IS_RATE {
2610            factor /= range_length_secs;
2611        }
2612        histogram = histogram.scale(factor).into_gauge();
2613        result.push(Some(histogram));
2614    }
2615
2616    Ok(ColumnarValue::Array(build_histogram_array(&result)))
2617}
2618
2619fn create_native_extrapolated_udf<const IS_COUNTER: bool, const IS_RATE: bool>(
2620    name: &'static str,
2621    collector: Option<PromqlAnnotationCollector>,
2622) -> ScalarUDF {
2623    let input_types = vec![
2624        RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
2625        RangeArray::convert_data_type(native_histogram_arrow_type()),
2626        DataType::Timestamp(TimeUnit::Millisecond, None),
2627        DataType::Int64,
2628    ];
2629    create_udf(
2630        name,
2631        input_types,
2632        native_histogram_arrow_type(),
2633        Volatility::Volatile,
2634        Arc::new(move |input: &[ColumnarValue]| {
2635            let range_length = extract_array(&input[3])?;
2636            let range_length = range_length
2637                .as_any()
2638                .downcast_ref::<Int64Array>()
2639                .ok_or_else(|| {
2640                    DataFusionError::Execution(format!(
2641                        "{name}: expect Int64 as range length type, found {}",
2642                        range_length.data_type()
2643                    ))
2644                })?;
2645            if range_length.is_empty() || range_length.is_null(0) {
2646                return Err(DataFusionError::Execution(format!(
2647                    "{name}: range length must contain a non-null Int64 value"
2648                )));
2649            }
2650            native_extrapolated_rate::<IS_COUNTER, IS_RATE>(
2651                input,
2652                range_length.value(0),
2653                name,
2654                collector.clone(),
2655            )
2656        }) as _,
2657    )
2658}
2659
2660pub struct NativeHistogramDelta;
2661pub struct NativeHistogramRate;
2662pub struct NativeHistogramIncrease;
2663
2664impl NativeHistogramDelta {
2665    pub const fn name() -> &'static str {
2666        "prom_native_histogram_delta"
2667    }
2668
2669    pub fn scalar_udf() -> ScalarUDF {
2670        Self::scalar_udf_with_collector(None)
2671    }
2672
2673    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2674        create_native_extrapolated_udf::<false, false>(Self::name(), collector)
2675    }
2676}
2677
2678impl NativeHistogramRate {
2679    pub const fn name() -> &'static str {
2680        "prom_native_histogram_rate"
2681    }
2682
2683    pub fn scalar_udf() -> ScalarUDF {
2684        Self::scalar_udf_with_collector(None)
2685    }
2686
2687    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2688        create_native_extrapolated_udf::<true, true>(Self::name(), collector)
2689    }
2690}
2691
2692impl NativeHistogramIncrease {
2693    pub const fn name() -> &'static str {
2694        "prom_native_histogram_increase"
2695    }
2696
2697    pub fn scalar_udf() -> ScalarUDF {
2698        Self::scalar_udf_with_collector(None)
2699    }
2700
2701    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2702        create_native_extrapolated_udf::<true, false>(Self::name(), collector)
2703    }
2704}
2705
2706fn native_idelta<const IS_RATE: bool>(
2707    input: &[ColumnarValue],
2708    func_name: &'static str,
2709    collector: Option<PromqlAnnotationCollector>,
2710) -> DfResult<ColumnarValue> {
2711    if input.len() != 2 {
2712        return Err(DataFusionError::Plan(format!(
2713            "{func_name} function should have 2 inputs"
2714        )));
2715    }
2716
2717    let ts_range = extract_range_dict(
2718        &input[0],
2719        func_name,
2720        "timestamp range vector",
2721        &DataType::Timestamp(TimeUnit::Millisecond, None),
2722    )?;
2723    let value_range = extract_range_dict(
2724        &input[1],
2725        func_name,
2726        "value range vector",
2727        &native_histogram_arrow_type(),
2728    )?;
2729
2730    if ts_range.keys().values() != value_range.keys().values() {
2731        return Err(DataFusionError::Execution(format!(
2732            "{func_name}: timestamp and value ranges should have the same window layout"
2733        )));
2734    }
2735
2736    let ts_values = ts_range
2737        .values()
2738        .as_any()
2739        .downcast_ref::<TimestampMillisecondArray>()
2740        .expect("validated timestamp range")
2741        .values();
2742    let histograms = value_range
2743        .values()
2744        .as_any()
2745        .downcast_ref::<StructArray>()
2746        .expect("validated native histogram range");
2747    let mut result = Vec::with_capacity(ts_range.keys().len());
2748
2749    for key in ts_range.keys().values() {
2750        let (offset, length) = unpack(*key);
2751        let offset = offset as usize;
2752        let length = length as usize;
2753        if length < 2 {
2754            result.push(None);
2755            continue;
2756        }
2757
2758        let mut samples = Vec::with_capacity(2);
2759        let mut has_null = false;
2760        for row in offset + length - 2..offset + length {
2761            let Some(histogram) = read_histogram(histograms, row)? else {
2762                has_null = true;
2763                break;
2764            };
2765            samples.push(histogram);
2766        }
2767        if has_null {
2768            result.push(None);
2769            continue;
2770        }
2771
2772        let wrong_flavor = samples.iter().any(|histogram| {
2773            if IS_RATE {
2774                histogram.reset_hint == GAUGE_RESET_HINT
2775            } else {
2776                histogram.reset_hint != GAUGE_RESET_HINT
2777            }
2778        });
2779        if wrong_flavor {
2780            let expected = if IS_RATE { "counter" } else { "gauge" };
2781            record_warning(
2782                &collector,
2783                format!("{func_name}: native histogram input should be a {expected} histogram"),
2784            );
2785        }
2786
2787        let sampled_interval_secs =
2788            (ts_values[offset + length - 1] - ts_values[offset + length - 2]) as f64 / 1000.0;
2789        if sampled_interval_secs <= 0.0 {
2790            result.push(None);
2791            continue;
2792        }
2793        record_custom_reconciliation(&collector, func_name, &samples[0], &samples[1]);
2794        let value = idelta_value(
2795            &samples,
2796            IS_RATE,
2797            ts_values[offset + length - 2],
2798            ts_values[offset + length - 1],
2799            sampled_interval_secs,
2800        );
2801        if value.is_none() {
2802            record_warning(
2803                &collector,
2804                format!("{func_name}: dropped native histogram range with incompatible schemas"),
2805            );
2806        }
2807        result.push(value);
2808    }
2809
2810    Ok(ColumnarValue::Array(build_histogram_array(&result)))
2811}
2812
2813fn create_native_idelta_udf<const IS_RATE: bool>(
2814    name: &'static str,
2815    collector: Option<PromqlAnnotationCollector>,
2816) -> ScalarUDF {
2817    create_udf(
2818        name,
2819        vec![
2820            RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
2821            RangeArray::convert_data_type(native_histogram_arrow_type()),
2822        ],
2823        native_histogram_arrow_type(),
2824        Volatility::Volatile,
2825        Arc::new(move |input: &[ColumnarValue]| {
2826            native_idelta::<IS_RATE>(input, name, collector.clone())
2827        }) as _,
2828    )
2829}
2830
2831pub struct NativeHistogramIDelta;
2832pub struct NativeHistogramIRate;
2833
2834impl NativeHistogramIDelta {
2835    pub const fn name() -> &'static str {
2836        "prom_native_histogram_idelta"
2837    }
2838
2839    pub fn scalar_udf() -> ScalarUDF {
2840        Self::scalar_udf_with_collector(None)
2841    }
2842
2843    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2844        create_native_idelta_udf::<false>(Self::name(), collector)
2845    }
2846}
2847
2848impl NativeHistogramIRate {
2849    pub const fn name() -> &'static str {
2850        "prom_native_histogram_irate"
2851    }
2852
2853    pub fn scalar_udf() -> ScalarUDF {
2854        Self::scalar_udf_with_collector(None)
2855    }
2856
2857    pub fn scalar_udf_with_collector(collector: Option<PromqlAnnotationCollector>) -> ScalarUDF {
2858        create_native_idelta_udf::<true>(Self::name(), collector)
2859    }
2860}
2861
2862#[cfg(test)]
2863mod tests {
2864    use datafusion::arrow::datatypes::Field;
2865    use datafusion_common::config::ConfigOptions;
2866    use datafusion_expr::ScalarFunctionArgs;
2867
2868    use super::*;
2869
2870    fn sample_histogram(count: f64, sum: f64, positive_buckets: Vec<f64>) -> NativeHistogram {
2871        NativeHistogram {
2872            schema: 0,
2873            zero_threshold: 0.0,
2874            sum,
2875            reset_hint: UNKNOWN_COUNTER_RESET_HINT,
2876            start_timestamp: None,
2877            custom_values: Vec::new(),
2878            positive_spans: vec![Span {
2879                offset: 0,
2880                length: positive_buckets.len() as i32,
2881            }],
2882            negative_spans: Vec::new(),
2883            count,
2884            zero_count: 0.0,
2885            positive_buckets,
2886            negative_buckets: Vec::new(),
2887        }
2888    }
2889
2890    fn run_scalar_udf(udf: ScalarUDF, input: Vec<ColumnarValue>) -> f64 {
2891        let result = run_udf(udf, input, DataType::Float64);
2892        extract_array(&result)
2893            .unwrap()
2894            .as_any()
2895            .downcast_ref::<Float64Array>()
2896            .unwrap()
2897            .value(0)
2898    }
2899
2900    fn run_udf(udf: ScalarUDF, input: Vec<ColumnarValue>, return_type: DataType) -> ColumnarValue {
2901        let arg_fields = input
2902            .iter()
2903            .enumerate()
2904            .map(|(idx, input)| Arc::new(Field::new(format!("arg_{idx}"), input.data_type(), true)))
2905            .collect();
2906        let args = ScalarFunctionArgs {
2907            args: input,
2908            arg_fields,
2909            number_rows: 1,
2910            return_field: Arc::new(Field::new("result", return_type, true)),
2911            config_options: Arc::new(ConfigOptions::default()),
2912        };
2913
2914        udf.invoke_with_args(args).unwrap()
2915    }
2916
2917    fn run_histogram_udf(udf: ScalarUDF, input: Vec<ColumnarValue>) -> NativeHistogram {
2918        let result = run_udf(udf, input, native_histogram_arrow_type());
2919        let array = extract_array(&result)
2920            .unwrap()
2921            .as_any()
2922            .downcast_ref::<StructArray>()
2923            .unwrap()
2924            .clone();
2925        read_histogram(&array, 0).unwrap().unwrap()
2926    }
2927
2928    fn evaluated_histogram(
2929        accumulator: &mut NativeHistogramAggregateAccumulator,
2930    ) -> NativeHistogram {
2931        let ScalarValue::Struct(array) = accumulator.evaluate().unwrap() else {
2932            panic!("native histogram accumulator returned a non-struct value");
2933        };
2934        read_histogram(&array, 0).unwrap().unwrap()
2935    }
2936
2937    fn histogram_range_input(values: Vec<Option<NativeHistogram>>) -> Vec<ColumnarValue> {
2938        let timestamps = Arc::new(TimestampMillisecondArray::from_iter(
2939            (0..values.len()).map(|idx| Some((idx as i64 + 1) * 1000)),
2940        ));
2941        let histograms = build_histogram_array(&values);
2942        let range = [(0, values.len() as u32)];
2943        let ts_range = RangeArray::from_ranges(timestamps, range).unwrap();
2944        let value_range = RangeArray::from_ranges(histograms, range).unwrap();
2945
2946        vec![
2947            ColumnarValue::Array(Arc::new(ts_range.into_dict())),
2948            ColumnarValue::Array(Arc::new(value_range.into_dict())),
2949        ]
2950    }
2951
2952    fn mixed_range_input(
2953        name: &str,
2954        floats: Vec<Option<f64>>,
2955        histograms: Vec<Option<NativeHistogram>>,
2956    ) -> Vec<ColumnarValue> {
2957        assert_eq!(floats.len(), histograms.len());
2958        let timestamps = Arc::new(TimestampMillisecondArray::from_iter(
2959            (0..floats.len()).map(|idx| Some((idx as i64 + 1) * 1000)),
2960        ));
2961        let floats = Arc::new(Float64Array::from(floats));
2962        let histograms = build_histogram_array(&histograms);
2963        let range = [(0, u32::try_from(floats.len()).unwrap())];
2964        vec![
2965            ColumnarValue::Scalar(ScalarValue::Utf8(Some(name.to_string()))),
2966            ColumnarValue::Array(Arc::new(
2967                RangeArray::from_ranges(timestamps, range)
2968                    .unwrap()
2969                    .into_dict(),
2970            )),
2971            ColumnarValue::Array(Arc::new(
2972                RangeArray::from_ranges(floats, range).unwrap().into_dict(),
2973            )),
2974            ColumnarValue::Array(Arc::new(
2975                RangeArray::from_ranges(histograms, range)
2976                    .unwrap()
2977                    .into_dict(),
2978            )),
2979        ]
2980    }
2981
2982    fn mixed_float_result(udf: ScalarUDF, input: Vec<ColumnarValue>) -> Option<f64> {
2983        let result = run_udf(udf, input, DataType::Float64);
2984        let result = extract_array(&result).unwrap();
2985        let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
2986        result.is_valid(0).then(|| result.value(0))
2987    }
2988
2989    fn mixed_histogram_result(
2990        udf: ScalarUDF,
2991        input: Vec<ColumnarValue>,
2992    ) -> Option<NativeHistogram> {
2993        let result = run_udf(udf, input, native_histogram_arrow_type());
2994        let result = extract_array(&result).unwrap();
2995        let result = result.as_any().downcast_ref::<StructArray>().unwrap();
2996        read_histogram(result, 0).unwrap()
2997    }
2998
2999    fn run_histogram_range_udf(
3000        udf: ScalarUDF,
3001        histograms: Vec<NativeHistogram>,
3002    ) -> NativeHistogram {
3003        run_histogram_udf(
3004            udf,
3005            histogram_range_input(histograms.into_iter().map(Some).collect()),
3006        )
3007    }
3008
3009    fn run_float_range_udf(
3010        udf: ScalarUDF,
3011        histograms: Vec<Option<NativeHistogram>>,
3012    ) -> Option<f64> {
3013        let result = run_udf(udf, histogram_range_input(histograms), DataType::Float64);
3014        let result = extract_array(&result).unwrap();
3015        let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
3016        (!result.is_null(0)).then(|| result.value(0))
3017    }
3018
3019    fn run_extrapolated_histogram_udf(
3020        udf: ScalarUDF,
3021        histograms: Vec<NativeHistogram>,
3022    ) -> NativeHistogram {
3023        let range_length = histograms.len() as i64 * 1000;
3024        let timestamps = (0..histograms.len())
3025            .map(|idx| (idx as i64 + 1) * 1000)
3026            .collect();
3027        extrapolated_histogram_result(udf, timestamps, histograms, range_length, range_length)
3028            .unwrap()
3029    }
3030
3031    fn extrapolated_histogram_result(
3032        udf: ScalarUDF,
3033        timestamps: Vec<i64>,
3034        histograms: Vec<NativeHistogram>,
3035        range_end: i64,
3036        range_length: i64,
3037    ) -> Option<NativeHistogram> {
3038        assert_eq!(timestamps.len(), histograms.len());
3039        let range = [(0, u32::try_from(histograms.len()).unwrap())];
3040        let timestamps = Arc::new(TimestampMillisecondArray::from(timestamps));
3041        let histograms =
3042            build_histogram_array(&histograms.into_iter().map(Some).collect::<Vec<_>>());
3043        let mut input = vec![
3044            ColumnarValue::Array(Arc::new(
3045                RangeArray::from_ranges(timestamps, range)
3046                    .unwrap()
3047                    .into_dict(),
3048            )),
3049            ColumnarValue::Array(Arc::new(
3050                RangeArray::from_ranges(histograms, range)
3051                    .unwrap()
3052                    .into_dict(),
3053            )),
3054        ];
3055        input.push(ColumnarValue::Array(Arc::new(
3056            TimestampMillisecondArray::from(vec![range_end]),
3057        )));
3058        input.push(ColumnarValue::Array(Arc::new(Int64Array::from(vec![
3059            range_length,
3060        ]))));
3061        let result = run_udf(udf, input, native_histogram_arrow_type());
3062        let result = extract_array(&result).unwrap();
3063        let result = result.as_any().downcast_ref::<StructArray>().unwrap();
3064        read_histogram(result, 0).unwrap()
3065    }
3066
3067    fn collected_warnings(collector: &PromqlAnnotationCollector) -> Vec<String> {
3068        let mut warnings = Vec::new();
3069        collector.append_to(&mut warnings, &mut Vec::new());
3070        warnings
3071    }
3072
3073    fn collected_infos(collector: &PromqlAnnotationCollector) -> Vec<String> {
3074        let mut infos = Vec::new();
3075        collector.append_to(&mut Vec::new(), &mut infos);
3076        infos
3077    }
3078
3079    #[test]
3080    fn mixed_ranges_follow_prometheus_sample_type_semantics() {
3081        let first = sample_histogram(1.0, 1.0, vec![1.0]);
3082        let second = sample_histogram(3.0, 3.0, vec![3.0]);
3083        let collector = PromqlAnnotationCollector::default();
3084
3085        let mut rate = mixed_range_input(
3086            "rate",
3087            vec![Some(1.0), None, Some(3.0)],
3088            vec![None, Some(first.clone()), None],
3089        );
3090        rate.push(ColumnarValue::Array(Arc::new(
3091            TimestampMillisecondArray::from(vec![3000]),
3092        )));
3093        rate.push(ColumnarValue::Array(Arc::new(Int64Array::from(vec![3000]))));
3094        assert_eq!(
3095            mixed_float_result(MixedRange::float_udf(Some(collector.clone())), rate.clone()),
3096            None
3097        );
3098        assert_eq!(
3099            mixed_histogram_result(MixedRange::histogram_udf(Some(collector.clone())), rate),
3100            None
3101        );
3102        assert!(
3103            collected_warnings(&collector)
3104                .iter()
3105                .any(|warning| warning.contains("mix of float and native histogram"))
3106        );
3107
3108        let pure_rate = |floats, histograms| {
3109            let mut input = mixed_range_input("rate", floats, histograms);
3110            input.push(ColumnarValue::Array(Arc::new(
3111                TimestampMillisecondArray::from(vec![3000]),
3112            )));
3113            input.push(ColumnarValue::Array(Arc::new(Int64Array::from(vec![3000]))));
3114            input
3115        };
3116        let pure_float = pure_rate(
3117            vec![Some(1.0), Some(2.0), Some(3.0)],
3118            vec![None, None, None],
3119        );
3120        assert_eq!(
3121            mixed_float_result(MixedRange::float_udf(None), pure_float.clone()),
3122            Some(1.0)
3123        );
3124        assert_eq!(
3125            mixed_histogram_result(MixedRange::histogram_udf(None), pure_float),
3126            None
3127        );
3128        let pure_histogram = pure_rate(
3129            vec![None, None, None],
3130            vec![
3131                Some(sample_histogram(1.0, 1.0, vec![1.0])),
3132                Some(sample_histogram(2.0, 2.0, vec![2.0])),
3133                Some(sample_histogram(3.0, 3.0, vec![3.0])),
3134            ],
3135        );
3136        assert_eq!(
3137            mixed_float_result(MixedRange::float_udf(None), pure_histogram.clone()),
3138            None
3139        );
3140        assert_eq!(
3141            mixed_histogram_result(MixedRange::histogram_udf(None), pure_histogram)
3142                .unwrap()
3143                .count,
3144            1.0
3145        );
3146
3147        let idelta = mixed_range_input(
3148            "idelta",
3149            vec![Some(10.0), None, None],
3150            vec![None, Some(first.clone()), Some(second.clone())],
3151        );
3152        assert_eq!(
3153            mixed_float_result(MixedRange::float_udf(None), idelta.clone()),
3154            None
3155        );
3156        assert_eq!(
3157            mixed_histogram_result(MixedRange::histogram_udf(None), idelta)
3158                .unwrap()
3159                .count,
3160            2.0
3161        );
3162
3163        let alternating = || {
3164            mixed_range_input(
3165                "changes",
3166                vec![Some(1.0), None, None, Some(1.0)],
3167                vec![None, Some(first.clone()), Some(first.clone()), None],
3168            )
3169        };
3170        assert_eq!(
3171            mixed_float_result(MixedRange::float_udf(None), alternating()),
3172            Some(2.0)
3173        );
3174        let mut resets = alternating();
3175        resets[0] = ColumnarValue::Scalar(ScalarValue::Utf8(Some("resets".to_string())));
3176        assert_eq!(
3177            mixed_float_result(MixedRange::float_udf(None), resets),
3178            Some(2.0)
3179        );
3180
3181        for (name, expected) in [
3182            ("count_over_time", Some(4.0)),
3183            ("present_over_time", Some(1.0)),
3184            ("absent_over_time", None),
3185        ] {
3186            let mut input = alternating();
3187            input[0] = ColumnarValue::Scalar(ScalarValue::Utf8(Some(name.to_string())));
3188            assert_eq!(
3189                mixed_float_result(MixedRange::float_udf(None), input),
3190                expected,
3191                "{name}"
3192            );
3193        }
3194
3195        let last = mixed_range_input(
3196            "last_over_time",
3197            vec![None, Some(4.0)],
3198            vec![Some(first.clone()), None],
3199        );
3200        assert_eq!(
3201            mixed_float_result(MixedRange::float_udf(None), last.clone()),
3202            Some(4.0)
3203        );
3204        assert_eq!(
3205            mixed_histogram_result(MixedRange::histogram_udf(None), last),
3206            None
3207        );
3208
3209        let collector = PromqlAnnotationCollector::default();
3210        let histogram_only_min =
3211            mixed_range_input("min_over_time", vec![None], vec![Some(first.clone())]);
3212        assert_eq!(
3213            mixed_float_result(
3214                MixedRange::float_udf(Some(collector.clone())),
3215                histogram_only_min,
3216            ),
3217            None
3218        );
3219        assert!(collected_infos(&collector).is_empty());
3220
3221        let min = mixed_range_input(
3222            "min_over_time",
3223            vec![Some(3.0), None, Some(1.0)],
3224            vec![None, Some(first), None],
3225        );
3226        assert_eq!(
3227            mixed_float_result(MixedRange::float_udf(Some(collector.clone())), min),
3228            Some(1.0)
3229        );
3230        assert!(
3231            collected_infos(&collector)
3232                .iter()
3233                .any(|info| info.contains("ignored native histogram"))
3234        );
3235    }
3236
3237    #[test]
3238    fn count_sum_and_avg_read_struct() {
3239        let histograms = vec![Some(sample_histogram(6.0, 10.0, vec![2.0, 4.0]))];
3240        let array = build_histogram_array(&histograms);
3241        let input = vec![ColumnarValue::Array(array)];
3242
3243        let count = run_scalar_udf(NativeHistogramCount::scalar_udf(), input.clone());
3244        assert_eq!(count, 6.0);
3245
3246        let sum = run_scalar_udf(NativeHistogramSum::scalar_udf(), input.clone());
3247        assert_eq!(sum, 10.0);
3248
3249        let avg = run_scalar_udf(NativeHistogramAvg::scalar_udf(), input);
3250        assert_eq!(avg, 10.0 / 6.0);
3251    }
3252
3253    #[test]
3254    fn quantile_uses_bucket_bounds() {
3255        let histogram = sample_histogram(6.0, 10.0, vec![2.0, 4.0]);
3256        assert_eq!(histogram.quantile(0.0), 0.5);
3257        assert!(histogram.quantile(0.5) > 1.0);
3258        assert!(histogram.quantile(0.5) < 2.0);
3259    }
3260
3261    #[test]
3262    fn comparison_observes_explicit_sparse_zero_buckets() {
3263        let mut left = sample_histogram(1.0, 1.0, vec![1.0, 0.0]);
3264        left.reset_hint = COUNTER_RESET_HINT;
3265        left.start_timestamp = Some(1000);
3266        let mut right = sample_histogram(1.0, 1.0, vec![1.0]);
3267        right.reset_hint = NOT_COUNTER_RESET_HINT;
3268        right.start_timestamp = Some(2000);
3269
3270        let result = run_udf(
3271            NativeHistogramEq::scalar_udf(),
3272            vec![
3273                ColumnarValue::Array(build_histogram_array(&[Some(left)])),
3274                ColumnarValue::Array(build_histogram_array(&[Some(right)])),
3275            ],
3276            DataType::Boolean,
3277        );
3278        let values = extract_array(&result).unwrap();
3279        let values = values.as_any().downcast_ref::<BooleanArray>().unwrap();
3280        assert!(!values.value(0));
3281    }
3282
3283    #[test]
3284    fn unary_minus_returns_gauge_histogram() {
3285        let result = run_histogram_udf(
3286            NativeHistogramNeg::scalar_udf(),
3287            vec![ColumnarValue::Array(build_histogram_array(&[Some(
3288                sample_histogram(2.0, 3.0, vec![2.0]),
3289            )]))],
3290        );
3291
3292        assert_eq!(result.reset_hint, GAUGE_RESET_HINT);
3293        assert_eq!(result.count, -2.0);
3294        assert_eq!(result.sum, -3.0);
3295        assert_eq!(result.positive_buckets, vec![-2.0]);
3296    }
3297
3298    #[test]
3299    fn histogram_over_time_functions_preserve_reset_hints() {
3300        let mut first = sample_histogram(1.0, 1.0, vec![1.0]);
3301        first.reset_hint = COUNTER_RESET_HINT;
3302        let mut second = sample_histogram(2.0, 2.0, vec![2.0]);
3303        second.reset_hint = COUNTER_RESET_HINT;
3304
3305        let result = run_histogram_range_udf(
3306            NativeHistogramSumOverTime::scalar_udf(),
3307            vec![first.clone(), second.clone()],
3308        );
3309        assert_eq!(result.reset_hint, COUNTER_RESET_HINT);
3310        assert_eq!(result.count, 3.0);
3311        assert_eq!(result.sum, 3.0);
3312        assert_eq!(result.positive_buckets, vec![3.0]);
3313
3314        let result = run_histogram_range_udf(
3315            NativeHistogramAvgOverTime::scalar_udf(),
3316            vec![first.clone(), second.clone()],
3317        );
3318        assert_eq!(result.reset_hint, COUNTER_RESET_HINT);
3319        assert_eq!(result.count, 1.5);
3320        assert_eq!(result.sum, 1.5);
3321        assert_eq!(result.positive_buckets, vec![1.5]);
3322
3323        let result = run_histogram_range_udf(
3324            NativeHistogramLastOverTime::scalar_udf(),
3325            vec![first, second],
3326        );
3327        assert_eq!(result.reset_hint, COUNTER_RESET_HINT);
3328        assert_eq!(result.count, 2.0);
3329        assert_eq!(result.sum, 2.0);
3330        assert_eq!(result.positive_buckets, vec![2.0]);
3331    }
3332
3333    #[test]
3334    fn histogram_averages_avoid_sum_overflow() {
3335        let large = sample_histogram(1.0e308, 1.0e308, vec![1.0e308]);
3336
3337        let range_average = run_histogram_range_udf(
3338            NativeHistogramAvgOverTime::scalar_udf(),
3339            vec![large.clone(), large.clone()],
3340        );
3341        assert_eq!(range_average.count, 1.0e308);
3342        assert_eq!(range_average.sum, 1.0e308);
3343        assert_eq!(range_average.positive_buckets, vec![1.0e308]);
3344
3345        let mut aggregate =
3346            NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3347        aggregate.push_histogram(large.clone(), 1).unwrap();
3348        aggregate.push_histogram(large, 1).unwrap();
3349        let aggregate_average = evaluated_histogram(&mut aggregate);
3350        assert_eq!(aggregate_average.count, 1.0e308);
3351        assert_eq!(aggregate_average.sum, 1.0e308);
3352        assert_eq!(aggregate_average.positive_buckets, vec![1.0e308]);
3353    }
3354
3355    #[test]
3356    fn histogram_average_partial_states_are_weighted() {
3357        let mut first =
3358            NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3359        first
3360            .push_histogram(sample_histogram(1.0, 1.0, vec![1.0]), 1)
3361            .unwrap();
3362        first
3363            .push_histogram(sample_histogram(3.0, 3.0, vec![3.0]), 1)
3364            .unwrap();
3365        let first_state = first
3366            .state()
3367            .unwrap()
3368            .into_iter()
3369            .map(|value| value.to_array_of_size(1).unwrap())
3370            .collect::<Vec<_>>();
3371
3372        let mut second =
3373            NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3374        second
3375            .push_histogram(sample_histogram(8.0, 8.0, vec![8.0]), 1)
3376            .unwrap();
3377        let second_state = second
3378            .state()
3379            .unwrap()
3380            .into_iter()
3381            .map(|value| value.to_array_of_size(1).unwrap())
3382            .collect::<Vec<_>>();
3383
3384        let mut merged =
3385            NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3386        merged.merge_batch(&first_state).unwrap();
3387        merged.merge_batch(&second_state).unwrap();
3388        let average = evaluated_histogram(&mut merged);
3389        assert_eq!(average.count, 4.0);
3390        assert_eq!(average.sum, 4.0);
3391        assert_eq!(average.positive_buckets, vec![4.0]);
3392    }
3393
3394    #[test]
3395    fn histogram_average_rejects_sample_count_overflow() {
3396        let mut aggregate =
3397            NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Avg, None);
3398        aggregate.value = Some(sample_histogram(1.0, 1.0, vec![1.0]));
3399        aggregate.count = u64::MAX;
3400
3401        let error = aggregate
3402            .push_histogram(sample_histogram(1.0, 1.0, vec![1.0]), 1)
3403            .unwrap_err();
3404        assert!(error.to_string().contains("sample count overflow"));
3405    }
3406
3407    #[test]
3408    fn presence_only_range_functions_preserve_null_semantics() {
3409        let first = sample_histogram(1.0, 1.0, vec![1.0]);
3410        let second = sample_histogram(2.0, 2.0, vec![2.0]);
3411        assert_eq!(
3412            run_float_range_udf(
3413                NativeHistogramCountOverTime::scalar_udf(),
3414                vec![Some(first.clone()), Some(second.clone())],
3415            ),
3416            Some(2.0)
3417        );
3418        assert_eq!(
3419            run_float_range_udf(
3420                NativeHistogramPresentOverTime::scalar_udf(),
3421                vec![Some(first.clone()), Some(second.clone())],
3422            ),
3423            Some(1.0)
3424        );
3425        assert_eq!(
3426            run_float_range_udf(
3427                NativeHistogramCountOverTime::scalar_udf(),
3428                vec![None, Some(second.clone())],
3429            ),
3430            None
3431        );
3432
3433        let result = run_udf(
3434            NativeHistogramLastOverTime::scalar_udf(),
3435            histogram_range_input(vec![None, Some(second)]),
3436            native_histogram_arrow_type(),
3437        );
3438        let result = extract_array(&result).unwrap();
3439        let result = result.as_any().downcast_ref::<StructArray>().unwrap();
3440        assert!(result.is_null(0));
3441    }
3442
3443    #[test]
3444    fn resets_counts_histogram_flavor_transitions() {
3445        let mut counter = sample_histogram(1.0, 1.0, vec![1.0]);
3446        counter.reset_hint = NOT_COUNTER_RESET_HINT;
3447        let mut gauge = sample_histogram(2.0, 2.0, vec![2.0]);
3448        gauge.reset_hint = GAUGE_RESET_HINT;
3449        assert_eq!(
3450            run_float_range_udf(
3451                NativeHistogramResets::scalar_udf(),
3452                vec![Some(counter), Some(gauge)],
3453            ),
3454            Some(1.0)
3455        );
3456
3457        let mut gauge = sample_histogram(1.0, 1.0, vec![1.0]);
3458        gauge.reset_hint = GAUGE_RESET_HINT;
3459        let mut counter = sample_histogram(2.0, 2.0, vec![2.0]);
3460        counter.reset_hint = NOT_COUNTER_RESET_HINT;
3461        assert_eq!(
3462            run_float_range_udf(
3463                NativeHistogramResets::scalar_udf(),
3464                vec![Some(gauge), Some(counter)],
3465            ),
3466            Some(1.0)
3467        );
3468    }
3469
3470    #[test]
3471    fn wrong_flavor_functions_record_warnings() {
3472        let mut gauge_first = sample_histogram(1.0, 1.0, vec![1.0]);
3473        gauge_first.reset_hint = GAUGE_RESET_HINT;
3474        let mut gauge_last = sample_histogram(2.0, 2.0, vec![2.0]);
3475        gauge_last.reset_hint = GAUGE_RESET_HINT;
3476        let mut counter_first = sample_histogram(1.0, 1.0, vec![1.0]);
3477        counter_first.reset_hint = NOT_COUNTER_RESET_HINT;
3478        let mut counter_last = sample_histogram(2.0, 2.0, vec![2.0]);
3479        counter_last.reset_hint = NOT_COUNTER_RESET_HINT;
3480        let collector = PromqlAnnotationCollector::default();
3481
3482        run_extrapolated_histogram_udf(
3483            NativeHistogramRate::scalar_udf_with_collector(Some(collector.clone())),
3484            vec![gauge_first.clone(), gauge_last.clone()],
3485        );
3486        run_extrapolated_histogram_udf(
3487            NativeHistogramDelta::scalar_udf_with_collector(Some(collector.clone())),
3488            vec![counter_first.clone(), counter_last.clone()],
3489        );
3490        run_histogram_range_udf(
3491            NativeHistogramIRate::scalar_udf_with_collector(Some(collector.clone())),
3492            vec![gauge_first, gauge_last],
3493        );
3494        run_histogram_range_udf(
3495            NativeHistogramIDelta::scalar_udf_with_collector(Some(collector.clone())),
3496            vec![counter_first, counter_last],
3497        );
3498
3499        let warnings = collected_warnings(&collector);
3500        for expected in [
3501            format!(
3502                "{}: native histogram input should be a counter histogram",
3503                NativeHistogramRate::name()
3504            ),
3505            format!(
3506                "{}: native histogram input should be a gauge histogram",
3507                NativeHistogramDelta::name()
3508            ),
3509            format!(
3510                "{}: native histogram input should be a counter histogram",
3511                NativeHistogramIRate::name()
3512            ),
3513            format!(
3514                "{}: native histogram input should be a gauge histogram",
3515                NativeHistogramIDelta::name()
3516            ),
3517        ] {
3518            assert!(warnings.contains(&expected), "missing warning: {expected}");
3519        }
3520    }
3521
3522    #[test]
3523    fn subtraction_records_reset_hint_contradictions_for_incompatible_histograms() {
3524        for incompatible in [false, true] {
3525            let mut left = sample_histogram(2.0, 2.0, vec![2.0]);
3526            left.reset_hint = COUNTER_RESET_HINT;
3527            let mut right = sample_histogram(1.0, 1.0, vec![1.0]);
3528            right.reset_hint = NOT_COUNTER_RESET_HINT;
3529            if incompatible {
3530                right.schema = CUSTOM_BUCKETS_SCHEMA;
3531                right.custom_values = vec![1.0];
3532            }
3533            let collector = PromqlAnnotationCollector::default();
3534
3535            run_udf(
3536                NativeHistogramSub::scalar_udf_with_collector(Some(collector.clone())),
3537                vec![
3538                    ColumnarValue::Array(build_histogram_array(&[Some(left)])),
3539                    ColumnarValue::Array(build_histogram_array(&[Some(right)])),
3540                ],
3541                native_histogram_arrow_type(),
3542            );
3543
3544            assert!(collected_warnings(&collector).contains(&format!(
3545                "{}: native histogram counter reset hints contradict",
3546                NativeHistogramSub::name()
3547            )));
3548        }
3549    }
3550
3551    #[test]
3552    fn counter_reset_hint_history_survives_folds_and_state_merges() {
3553        let mut reset = sample_histogram(1.0, 1.0, vec![1.0]);
3554        reset.reset_hint = COUNTER_RESET_HINT;
3555        let mut unknown = sample_histogram(2.0, 2.0, vec![2.0]);
3556        unknown.reset_hint = UNKNOWN_COUNTER_RESET_HINT;
3557        let mut not_reset = sample_histogram(3.0, 3.0, vec![3.0]);
3558        not_reset.reset_hint = NOT_COUNTER_RESET_HINT;
3559
3560        let range_collector = PromqlAnnotationCollector::default();
3561        assert!(
3562            range_fold_histograms(
3563                vec![reset.clone(), unknown.clone(), not_reset.clone()],
3564                NativeHistogramAggregateKind::Sum,
3565                NativeHistogramSumOverTime::name(),
3566                &Some(range_collector.clone()),
3567            )
3568            .is_some()
3569        );
3570        assert!(collected_warnings(&range_collector).contains(&format!(
3571            "{}: native histogram counter reset hints contradict",
3572            NativeHistogramSumOverTime::name()
3573        )));
3574
3575        for kind in [
3576            NativeHistogramAggregateKind::Sum,
3577            NativeHistogramAggregateKind::Avg,
3578        ] {
3579            let mut first_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3580            first_partial.push_histogram(reset.clone(), 1).unwrap();
3581            first_partial.push_histogram(unknown.clone(), 1).unwrap();
3582            let first_states = first_partial
3583                .state()
3584                .unwrap()
3585                .into_iter()
3586                .map(|value| value.to_array_of_size(1).unwrap())
3587                .collect::<Vec<_>>();
3588
3589            let mut second_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3590            second_partial.push_histogram(not_reset.clone(), 1).unwrap();
3591            let second_states = second_partial
3592                .state()
3593                .unwrap()
3594                .into_iter()
3595                .map(|value| value.to_array_of_size(1).unwrap())
3596                .collect::<Vec<_>>();
3597
3598            let collector = PromqlAnnotationCollector::default();
3599            let mut merged =
3600                NativeHistogramAggregateAccumulator::new(kind, Some(collector.clone()));
3601            merged.merge_batch(&first_states).unwrap();
3602            merged.merge_batch(&second_states).unwrap();
3603            assert!(collected_warnings(&collector).contains(&format!(
3604                "{}: native histogram counter reset hints contradict",
3605                kind.name()
3606            )));
3607        }
3608    }
3609
3610    #[test]
3611    fn incompatible_aggregates_do_not_hide_reset_hint_contradictions() {
3612        let mut reset = sample_histogram(1.0, 1.0, vec![1.0]);
3613        reset.reset_hint = COUNTER_RESET_HINT;
3614        let mut incompatible_reset = sample_histogram(2.0, 2.0, vec![2.0]);
3615        incompatible_reset.schema = CUSTOM_BUCKETS_SCHEMA;
3616        incompatible_reset.custom_values = vec![1.0];
3617        incompatible_reset.reset_hint = COUNTER_RESET_HINT;
3618        let mut not_reset = sample_histogram(3.0, 3.0, vec![3.0]);
3619        not_reset.reset_hint = NOT_COUNTER_RESET_HINT;
3620
3621        for kind in [
3622            NativeHistogramAggregateKind::Sum,
3623            NativeHistogramAggregateKind::Avg,
3624        ] {
3625            for histograms in [
3626                [reset.clone(), incompatible_reset.clone(), not_reset.clone()],
3627                [reset.clone(), not_reset.clone(), incompatible_reset.clone()],
3628            ] {
3629                let collector = PromqlAnnotationCollector::default();
3630                let mut aggregate =
3631                    NativeHistogramAggregateAccumulator::new(kind, Some(collector.clone()));
3632                for histogram in histograms {
3633                    aggregate.push_histogram(histogram, 1).unwrap();
3634                }
3635
3636                assert!(aggregate.dropped_incompatible);
3637                let warnings = collected_warnings(&collector);
3638                assert!(warnings.contains(&format!(
3639                    "{}: dropped native histogram aggregate with incompatible schemas",
3640                    kind.name()
3641                )));
3642                assert!(warnings.contains(&format!(
3643                    "{}: native histogram counter reset hints contradict",
3644                    kind.name()
3645                )));
3646            }
3647
3648            let mut dropped_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3649            dropped_partial.push_histogram(reset.clone(), 1).unwrap();
3650            dropped_partial
3651                .push_histogram(incompatible_reset.clone(), 1)
3652                .unwrap();
3653            let dropped_states = dropped_partial
3654                .state()
3655                .unwrap()
3656                .into_iter()
3657                .map(|value| value.to_array_of_size(1).unwrap())
3658                .collect::<Vec<_>>();
3659
3660            let mut opposing_partial = NativeHistogramAggregateAccumulator::new(kind, None);
3661            opposing_partial
3662                .push_histogram(not_reset.clone(), 1)
3663                .unwrap();
3664            let opposing_states = opposing_partial
3665                .state()
3666                .unwrap()
3667                .into_iter()
3668                .map(|value| value.to_array_of_size(1).unwrap())
3669                .collect::<Vec<_>>();
3670
3671            for (first, second) in [
3672                (&dropped_states, &opposing_states),
3673                (&opposing_states, &dropped_states),
3674            ] {
3675                let collector = PromqlAnnotationCollector::default();
3676                let mut merged =
3677                    NativeHistogramAggregateAccumulator::new(kind, Some(collector.clone()));
3678                merged.merge_batch(first).unwrap();
3679                merged.merge_batch(second).unwrap();
3680
3681                assert!(merged.dropped_incompatible);
3682                assert!(collected_warnings(&collector).contains(&format!(
3683                    "{}: native histogram counter reset hints contradict",
3684                    kind.name()
3685                )));
3686            }
3687        }
3688
3689        for (kind, name) in [
3690            (
3691                NativeHistogramAggregateKind::Sum,
3692                NativeHistogramSumOverTime::name(),
3693            ),
3694            (
3695                NativeHistogramAggregateKind::Avg,
3696                NativeHistogramAvgOverTime::name(),
3697            ),
3698        ] {
3699            let collector = PromqlAnnotationCollector::default();
3700            assert!(
3701                range_fold_histograms(
3702                    vec![reset.clone(), incompatible_reset.clone(), not_reset.clone()],
3703                    kind,
3704                    name,
3705                    &Some(collector.clone()),
3706                )
3707                .is_none()
3708            );
3709            let warnings = collected_warnings(&collector);
3710            assert!(warnings.contains(&format!(
3711                "{name}: dropped native histogram range with incompatible schemas"
3712            )));
3713            assert!(warnings.contains(&format!(
3714                "{name}: native histogram counter reset hints contradict"
3715            )));
3716        }
3717    }
3718
3719    #[test]
3720    fn histogram_aggregate_accumulator_accounts_for_heap_allocations() {
3721        let histogram = sample_histogram(3.0, 3.0, vec![1.0, 2.0]);
3722        let heap_size = histogram.custom_values.capacity() * size_of::<f64>()
3723            + histogram.positive_spans.capacity() * size_of::<Span>()
3724            + histogram.negative_spans.capacity() * size_of::<Span>()
3725            + histogram.positive_buckets.capacity() * size_of::<f64>()
3726            + histogram.negative_buckets.capacity() * size_of::<f64>();
3727        let mut accumulator =
3728            NativeHistogramAggregateAccumulator::new(NativeHistogramAggregateKind::Sum, None);
3729        let empty_size = accumulator.size();
3730
3731        accumulator.push_histogram(histogram, 1).unwrap();
3732
3733        assert!(heap_size > 0);
3734        assert_eq!(accumulator.size(), empty_size + heap_size);
3735    }
3736
3737    #[test]
3738    fn absent_over_time_handles_histogram_ranges() {
3739        let values = vec![Some(sample_histogram(1.0, 1.0, vec![1.0]))];
3740        let timestamps = Arc::new(TimestampMillisecondArray::from_iter([Some(1000)]));
3741        let histograms = build_histogram_array(&values);
3742        let ranges = [(0, 1), (0, 0)];
3743        let ts_range = RangeArray::from_ranges(timestamps, ranges).unwrap();
3744        let value_range = RangeArray::from_ranges(histograms, ranges).unwrap();
3745
3746        let result = run_udf(
3747            NativeHistogramAbsentOverTime::scalar_udf(),
3748            vec![
3749                ColumnarValue::Array(Arc::new(ts_range.into_dict())),
3750                ColumnarValue::Array(Arc::new(value_range.into_dict())),
3751            ],
3752            DataType::Float64,
3753        );
3754        let result = extract_array(&result).unwrap();
3755        let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
3756
3757        assert!(result.is_null(0));
3758        assert_eq!(result.value(1), 1.0);
3759    }
3760
3761    #[test]
3762    fn delta_requires_exact_layout() {
3763        let first = sample_histogram(2.0, 3.0, vec![1.0, 1.0]);
3764        let last = sample_histogram(5.0, 8.0, vec![2.0, 3.0]);
3765        let delta = histogram_delta(&[first, last], &[0, 1], false).unwrap();
3766        assert_eq!(delta.count, 3.0);
3767        assert_eq!(delta.sum, 5.0);
3768        assert_eq!(delta.reset_hint, GAUGE_RESET_HINT);
3769        assert_eq!(delta.positive_buckets, vec![1.0, 2.0]);
3770    }
3771
3772    #[test]
3773    fn reset_hint_shortcuts_detection() {
3774        let previous = sample_histogram(6.0, 10.0, vec![2.0, 4.0]);
3775
3776        let mut current = sample_histogram(7.0, 12.0, vec![3.0, 4.0]);
3777        current.reset_hint = COUNTER_RESET_HINT;
3778        assert!(current.detect_reset(&previous));
3779
3780        let mut current = sample_histogram(5.0, 8.0, vec![1.0, 4.0]);
3781        current.reset_hint = NOT_COUNTER_RESET_HINT;
3782        assert!(!current.detect_reset(&previous));
3783    }
3784
3785    #[test]
3786    fn start_timestamp_detects_counter_reset() {
3787        let first = sample_histogram(6.0, 10.0, vec![2.0, 4.0]);
3788        let mut last = sample_histogram(7.0, 12.0, vec![3.0, 4.0]);
3789        last.start_timestamp = Some(1500);
3790
3791        let delta = histogram_delta(&[first.clone(), last.clone()], &[1000, 2000], true).unwrap();
3792        assert_eq!(delta.count, 7.0);
3793        assert_eq!(delta.sum, 12.0);
3794
3795        let idelta = idelta_value(&[first, last], true, 1000, 2000, 1.0).unwrap();
3796        assert_eq!(idelta.count, 7.0);
3797        assert_eq!(idelta.sum, 12.0);
3798    }
3799
3800    #[test]
3801    fn extrapolated_rate_uses_start_timestamp_synthetic_zero() {
3802        let mut single = sample_histogram(1.0, 1.0, vec![1.0]);
3803        single.start_timestamp = Some(1_000);
3804
3805        let rate = extrapolated_histogram_result(
3806            NativeHistogramRate::scalar_udf(),
3807            vec![2_000],
3808            vec![single.clone()],
3809            3_000,
3810            3_000,
3811        )
3812        .unwrap();
3813        assert_eq!(rate.count, 1.0 / 3.0);
3814        assert_eq!(rate.sum, 1.0 / 3.0);
3815
3816        let increase = extrapolated_histogram_result(
3817            NativeHistogramIncrease::scalar_udf(),
3818            vec![2_000],
3819            vec![single],
3820            3_000,
3821            3_000,
3822        )
3823        .unwrap();
3824        assert_eq!(increase.count, 1.0);
3825        assert_eq!(increase.sum, 1.0);
3826
3827        let mut first = sample_histogram(2.0, 2.0, vec![2.0]);
3828        first.start_timestamp = Some(1_000);
3829        let last = sample_histogram(4.0, 4.0, vec![4.0]);
3830        let increase = extrapolated_histogram_result(
3831            NativeHistogramIncrease::scalar_udf(),
3832            vec![2_000, 3_000],
3833            vec![first, last],
3834            3_000,
3835            3_000,
3836        )
3837        .unwrap();
3838        assert_eq!(increase.count, 4.0);
3839        assert_eq!(increase.sum, 4.0);
3840    }
3841
3842    #[test]
3843    fn extrapolated_rate_requires_strictly_in_range_start_timestamp() {
3844        for (start_timestamp, range_end, range_length) in [
3845            (0, 3_000, 3_000),
3846            (500, 3_000, 2_000),
3847            (1_000, 3_000, 2_000),
3848            (2_000, 3_000, 3_000),
3849            (2_500, 3_000, 3_000),
3850        ] {
3851            let mut sample = sample_histogram(1.0, 1.0, vec![1.0]);
3852            sample.start_timestamp = Some(start_timestamp);
3853            assert!(
3854                extrapolated_histogram_result(
3855                    NativeHistogramRate::scalar_udf(),
3856                    vec![2_000],
3857                    vec![sample],
3858                    range_end,
3859                    range_length,
3860                )
3861                .is_none(),
3862                "start_timestamp={start_timestamp}"
3863            );
3864        }
3865
3866        let mut gauge = sample_histogram(1.0, 1.0, vec![1.0]);
3867        gauge.start_timestamp = Some(1_000);
3868        gauge.reset_hint = GAUGE_RESET_HINT;
3869        assert!(
3870            extrapolated_histogram_result(
3871                NativeHistogramDelta::scalar_udf(),
3872                vec![2_000],
3873                vec![gauge],
3874                3_000,
3875                3_000,
3876            )
3877            .is_none()
3878        );
3879    }
3880
3881    #[test]
3882    fn first_reset_ignores_incompatible_pre_reset_layout() {
3883        let first = sample_histogram(10.0, 10.0, vec![10.0]);
3884        let second = NativeHistogram {
3885            schema: CUSTOM_BUCKETS_SCHEMA,
3886            zero_threshold: 0.0,
3887            sum: 2.0,
3888            reset_hint: COUNTER_RESET_HINT,
3889            start_timestamp: None,
3890            custom_values: vec![1.0],
3891            positive_spans: vec![Span {
3892                offset: 0,
3893                length: 1,
3894            }],
3895            negative_spans: Vec::new(),
3896            count: 2.0,
3897            zero_count: 0.0,
3898            positive_buckets: vec![2.0],
3899            negative_buckets: Vec::new(),
3900        };
3901
3902        let delta = histogram_delta(&[first, second], &[1_000, 2_000], true).unwrap();
3903        assert_eq!(delta.schema, CUSTOM_BUCKETS_SCHEMA);
3904        assert_eq!(delta.count, 2.0);
3905        assert_eq!(delta.sum, 2.0);
3906        assert_eq!(delta.positive_buckets, vec![2.0]);
3907    }
3908
3909    #[test]
3910    fn counter_delta_handles_reset_segment_boundaries() {
3911        let first = sample_histogram(5.0, 5.0, vec![5.0]);
3912        let second = sample_histogram(7.0, 7.0, vec![7.0]);
3913        let mut reset = sample_histogram(2.0, 2.0, vec![2.0]);
3914        reset.reset_hint = COUNTER_RESET_HINT;
3915        let last = sample_histogram(4.0, 4.0, vec![4.0]);
3916
3917        let delta = histogram_delta(
3918            &[first, second, reset, last],
3919            &[1_000, 2_000, 3_000, 4_000],
3920            true,
3921        )
3922        .unwrap();
3923        assert_eq!(delta.count, 6.0);
3924        assert_eq!(delta.sum, 6.0);
3925        assert_eq!(delta.positive_buckets, vec![6.0]);
3926    }
3927}