Skip to main content

common_query/
native_histogram.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Shared native histogram field contract.
16//!
17//! Prom remote-write v2 stores these names as children of one Struct field, while
18//! metric-engine uses the same contract to recognize native histogram tables.
19//! [`NativeHistogram`] is the query-time representation and therefore normalizes
20//! integer and floating-point payloads to absolute `f64` counts.
21
22use std::collections::BTreeMap;
23use std::sync::Arc;
24
25use datafusion::arrow::array::{
26    Array, ArrayRef, Float64Array, Int32Array, ListArray, PrimitiveArray, StructArray,
27    TimestampMillisecondArray, UInt64Array,
28};
29use datafusion::arrow::buffer::NullBuffer;
30use datafusion::arrow::datatypes::{
31    ArrowPrimitiveType, DataType as ArrowDataType, Field, Float64Type, Int32Type, Int64Type,
32    TimestampMillisecondType, UInt32Type, UInt64Type,
33};
34use datafusion_common::{DataFusionError, Result as DfResult};
35use datatypes::data_type::{ConcreteDataType, DataType};
36use datatypes::types::{StructField, StructType};
37use once_cell::sync::Lazy;
38
39use crate::prelude::greptime_native_histogram;
40use crate::prometheus::format_prometheus_float;
41
42pub const NATIVE_HISTOGRAM_FIELD: &str = "greptime_native_histogram";
43pub const SCHEMA_FIELD: &str = "schema";
44pub const ZERO_THRESHOLD_FIELD: &str = "zero_threshold";
45pub const SUM_FIELD: &str = "sum";
46pub const RESET_HINT_FIELD: &str = "reset_hint";
47pub const START_TIMESTAMP_FIELD: &str = "start_timestamp";
48pub const CUSTOM_VALUES_FIELD: &str = "custom_values";
49pub const POSITIVE_SPAN_OFFSETS_FIELD: &str = "positive_span_offsets";
50pub const POSITIVE_SPAN_LENGTHS_FIELD: &str = "positive_span_lengths";
51pub const NEGATIVE_SPAN_OFFSETS_FIELD: &str = "negative_span_offsets";
52pub const NEGATIVE_SPAN_LENGTHS_FIELD: &str = "negative_span_lengths";
53pub const COUNT_U64_FIELD: &str = "count_u64";
54pub const ZERO_COUNT_U64_FIELD: &str = "zero_count_u64";
55pub const POSITIVE_BUCKETS_I64_FIELD: &str = "positive_buckets_i64";
56pub const NEGATIVE_BUCKETS_I64_FIELD: &str = "negative_buckets_i64";
57pub const COUNT_F64_FIELD: &str = "count_f64";
58pub const ZERO_COUNT_F64_FIELD: &str = "zero_count_f64";
59pub const POSITIVE_BUCKETS_F64_FIELD: &str = "positive_buckets_f64";
60pub const NEGATIVE_BUCKETS_F64_FIELD: &str = "negative_buckets_f64";
61
62// Keep int and float payloads in separate columns. The populated family is the
63// type signal, so we don't need to persist an extra histogram-type tag.
64pub const NATIVE_HISTOGRAM_FIELD_NAMES: &[&str] = &[
65    SCHEMA_FIELD,
66    ZERO_THRESHOLD_FIELD,
67    SUM_FIELD,
68    RESET_HINT_FIELD,
69    START_TIMESTAMP_FIELD,
70    CUSTOM_VALUES_FIELD,
71    POSITIVE_SPAN_OFFSETS_FIELD,
72    POSITIVE_SPAN_LENGTHS_FIELD,
73    NEGATIVE_SPAN_OFFSETS_FIELD,
74    NEGATIVE_SPAN_LENGTHS_FIELD,
75    COUNT_U64_FIELD,
76    ZERO_COUNT_U64_FIELD,
77    POSITIVE_BUCKETS_I64_FIELD,
78    NEGATIVE_BUCKETS_I64_FIELD,
79    COUNT_F64_FIELD,
80    ZERO_COUNT_F64_FIELD,
81    POSITIVE_BUCKETS_F64_FIELD,
82    NEGATIVE_BUCKETS_F64_FIELD,
83];
84
85static NATIVE_HISTOGRAM_VALUE_TYPE: Lazy<ConcreteDataType> = Lazy::new(|| {
86    let fields = NATIVE_HISTOGRAM_FIELD_NAMES
87        .iter()
88        .filter_map(|name| {
89            let data_type = native_histogram_field_type(name)?;
90            Some(StructField::new((*name).to_string(), data_type, true))
91        })
92        .collect();
93    ConcreteDataType::struct_datatype(StructType::new(Arc::new(fields)))
94});
95
96/// Returns the exact Greptime type for a persisted native histogram field.
97pub fn native_histogram_field_type(name: &str) -> Option<ConcreteDataType> {
98    match name {
99        SCHEMA_FIELD | RESET_HINT_FIELD => Some(ConcreteDataType::int32_datatype()),
100        ZERO_THRESHOLD_FIELD | SUM_FIELD | COUNT_F64_FIELD | ZERO_COUNT_F64_FIELD => {
101            Some(ConcreteDataType::float64_datatype())
102        }
103        START_TIMESTAMP_FIELD => Some(ConcreteDataType::timestamp_millisecond_datatype()),
104        CUSTOM_VALUES_FIELD | POSITIVE_BUCKETS_F64_FIELD | NEGATIVE_BUCKETS_F64_FIELD => Some(
105            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::float64_datatype())),
106        ),
107        POSITIVE_SPAN_OFFSETS_FIELD | NEGATIVE_SPAN_OFFSETS_FIELD => Some(
108            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype())),
109        ),
110        POSITIVE_SPAN_LENGTHS_FIELD | NEGATIVE_SPAN_LENGTHS_FIELD => Some(
111            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::uint32_datatype())),
112        ),
113        COUNT_U64_FIELD | ZERO_COUNT_U64_FIELD => Some(ConcreteDataType::uint64_datatype()),
114        POSITIVE_BUCKETS_I64_FIELD | NEGATIVE_BUCKETS_I64_FIELD => Some(
115            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int64_datatype())),
116        ),
117        _ => None,
118    }
119}
120
121/// Returns the exact Greptime type for a complete native histogram value.
122pub fn native_histogram_value_type() -> &'static ConcreteDataType {
123    &NATIVE_HISTOGRAM_VALUE_TYPE
124}
125
126/// Returns whether `data_type` matches the native histogram value contract.
127pub fn is_native_histogram_value_type(data_type: &ConcreteDataType) -> bool {
128    data_type == native_histogram_value_type()
129}
130
131/// Returns whether a named column matches the configured native histogram contract.
132pub fn is_native_histogram_value_schema(name: &str, data_type: &ConcreteDataType) -> bool {
133    name == greptime_native_histogram() && is_native_histogram_value_type(data_type)
134}
135
136/// Schema identifier for native histograms with explicit custom bucket bounds.
137pub const CUSTOM_BUCKETS_SCHEMA: i32 = -53;
138const MIN_EXPONENTIAL_SCHEMA: i32 = -4;
139const MAX_EXPONENTIAL_SCHEMA: i32 = 8;
140
141/// Information carried by a native histogram about a possible counter reset.
142///
143/// The persisted and protobuf representation remains an `i32`. Unrecognized
144/// values are retained so data written by a newer producer can still round-trip.
145#[derive(Clone, Copy, Debug, PartialEq, Eq)]
146pub enum CounterResetHint {
147    /// The payload must be inspected to determine whether a reset occurred.
148    Unknown,
149    /// This is the first histogram after a counter reset.
150    CounterReset,
151    /// No counter reset occurred since the previous histogram.
152    NotCounterReset,
153    /// This is a gauge histogram, so counter resets do not apply.
154    Gauge,
155    /// A value not understood by this version.
156    Unrecognized(i32),
157}
158
159impl From<i32> for CounterResetHint {
160    fn from(value: i32) -> Self {
161        match value {
162            0 => Self::Unknown,
163            1 => Self::CounterReset,
164            2 => Self::NotCounterReset,
165            3 => Self::Gauge,
166            value => Self::Unrecognized(value),
167        }
168    }
169}
170
171impl From<CounterResetHint> for i32 {
172    fn from(value: CounterResetHint) -> Self {
173        match value {
174            CounterResetHint::Unknown => 0,
175            CounterResetHint::CounterReset => 1,
176            CounterResetHint::NotCounterReset => 2,
177            CounterResetHint::Gauge => 3,
178            CounterResetHint::Unrecognized(value) => value,
179        }
180    }
181}
182
183/// Reset hint indicating that the payload must be inspected.
184pub const UNKNOWN_COUNTER_RESET_HINT: CounterResetHint = CounterResetHint::Unknown;
185/// Reset hint indicating the first histogram after a counter reset.
186pub const COUNTER_RESET_HINT: CounterResetHint = CounterResetHint::CounterReset;
187/// Reset hint indicating that no counter reset occurred.
188pub const NOT_COUNTER_RESET_HINT: CounterResetHint = CounterResetHint::NotCounterReset;
189/// Reset hint identifying a gauge histogram.
190pub const GAUGE_RESET_HINT: CounterResetHint = CounterResetHint::Gauge;
191
192/// A contiguous run of populated sparse buckets.
193#[derive(Clone, Debug, PartialEq)]
194pub struct Span {
195    /// The first bucket index, or the gap after the preceding span.
196    pub offset: i32,
197    /// Number of consecutive buckets in the span.
198    pub length: u32,
199}
200
201/// Inclusion rules for a materialized bucket's lower and upper bounds.
202#[derive(Clone, Copy, Debug, PartialEq, Eq)]
203enum BoundaryRule {
204    OpenLeft = 0,
205    OpenRight = 1,
206    ClosedBoth = 3,
207}
208
209/// A materialized bucket used by query-time estimators and renderers.
210#[derive(Clone, Debug, PartialEq)]
211struct Bucket {
212    lower: f64,
213    upper: f64,
214    count: f64,
215    boundary_rule: BoundaryRule,
216}
217
218/// Query-time representation of a Prometheus native histogram.
219///
220/// Bucket counts are absolute `f64` values even when the persisted payload used
221/// integer counts or deltas.
222#[derive(Clone, Debug, PartialEq)]
223pub struct NativeHistogram {
224    /// Exponential bucket schema, or [`CUSTOM_BUCKETS_SCHEMA`].
225    pub schema: i32,
226    /// Absolute width of the zero bucket.
227    pub zero_threshold: f64,
228    /// Sum of all observations.
229    pub sum: f64,
230    /// Counter reset information associated with this sample.
231    pub reset_hint: CounterResetHint,
232    /// Optional time in milliseconds when this histogram started counting.
233    pub start_timestamp: Option<i64>,
234    /// Inclusive upper bounds used when `schema` is [`CUSTOM_BUCKETS_SCHEMA`].
235    pub custom_values: Vec<f64>,
236    /// Sparse spans describing positive or custom buckets.
237    pub positive_spans: Vec<Span>,
238    /// Sparse spans describing negative buckets.
239    pub negative_spans: Vec<Span>,
240    /// Total number of observations.
241    pub count: f64,
242    /// Number of observations in the zero bucket.
243    pub zero_count: f64,
244    /// Absolute counts for positive or custom buckets.
245    pub positive_buckets: Vec<f64>,
246    /// Absolute counts for negative buckets.
247    pub negative_buckets: Vec<f64>,
248}
249
250impl NativeHistogram {
251    fn uses_custom_buckets(&self) -> bool {
252        self.schema == CUSTOM_BUCKETS_SCHEMA
253    }
254
255    fn compatible_with(&self, other: &Self) -> bool {
256        self.schema == other.schema
257            && self.zero_threshold == other.zero_threshold
258            && self.custom_values == other.custom_values
259    }
260
261    /// Returns a histogram with the same layout and metadata but zeroed payload values.
262    pub fn zero_like(&self) -> Self {
263        let mut result = self.clone();
264        result.count = 0.0;
265        result.zero_count = 0.0;
266        result.sum = 0.0;
267        result.positive_buckets.fill(0.0);
268        result.negative_buckets.fill(0.0);
269        result
270    }
271
272    fn combine_exact(
273        &self,
274        other: &Self,
275        reset_hint: CounterResetHint,
276        op: impl Fn(f64, f64) -> f64 + Copy,
277    ) -> Option<Self> {
278        if !self.compatible_with(other) {
279            return None;
280        }
281
282        let mut result = self.clone();
283        result.count = op(result.count, other.count);
284        result.zero_count = op(result.zero_count, other.zero_count);
285        result.sum = op(result.sum, other.sum);
286        result.reset_hint = reset_hint;
287        (result.positive_spans, result.positive_buckets) = merge_side(
288            &self.positive_spans,
289            &self.positive_buckets,
290            &other.positive_spans,
291            &other.positive_buckets,
292            op,
293        )?;
294        (result.negative_spans, result.negative_buckets) = merge_side(
295            &self.negative_spans,
296            &self.negative_buckets,
297            &other.negative_spans,
298            &other.negative_buckets,
299            op,
300        )?;
301        Some(result)
302    }
303
304    /// Adds two histograms after reconciling compatible bucket layouts.
305    ///
306    /// Returns `None` when the layouts are invalid or cannot be reconciled.
307    pub fn add(&self, other: &Self) -> Option<Self> {
308        let reset_hint = add_reset_hint(self.reset_hint, other.reset_hint);
309        let (left, right) = self.reconcile(other)?;
310        left.combine_exact(&right, reset_hint, |left, right| left + right)?
311            .compact()
312    }
313
314    /// Subtracts `other` after reconciling compatible bucket layouts.
315    ///
316    /// Returns `None` when the layouts are invalid or cannot be reconciled.
317    pub fn sub(&self, other: &Self) -> Option<Self> {
318        let (left, right) = self.reconcile(other)?;
319        left.combine_exact(&right, CounterResetHint::Gauge, |left, right| left - right)?
320            .compact()
321    }
322
323    /// Negates every payload value and marks the result as a gauge histogram.
324    pub fn negated(self) -> Self {
325        self.scale(-1.0)
326    }
327
328    /// Marks this histogram as a gauge without changing its payload.
329    pub fn into_gauge(mut self) -> Self {
330        self.reset_hint = CounterResetHint::Gauge;
331        self
332    }
333
334    /// Returns whether the histograms carry explicit, contradictory reset hints.
335    pub fn counter_reset_hints_contradict(&self, other: &Self) -> bool {
336        matches!(
337            (self.reset_hint, other.reset_hint),
338            (
339                CounterResetHint::CounterReset,
340                CounterResetHint::NotCounterReset
341            ) | (
342                CounterResetHint::NotCounterReset,
343                CounterResetHint::CounterReset
344            )
345        )
346    }
347
348    /// Returns whether two custom histograms use different bucket bounds.
349    pub fn needs_custom_reconciliation(&self, other: &Self) -> bool {
350        self.uses_custom_buckets()
351            && other.uses_custom_buckets()
352            && self.custom_values != other.custom_values
353    }
354
355    /// Compares PromQL-visible payload values by their exact bit patterns.
356    ///
357    /// Reset hints and start timestamps are ignored.
358    pub fn promql_eq(&self, other: &Self) -> bool {
359        self.schema == other.schema
360            && self.zero_threshold == other.zero_threshold
361            && self.custom_values == other.custom_values
362            && self.count.to_bits() == other.count.to_bits()
363            && self.zero_count.to_bits() == other.zero_count.to_bits()
364            && self.sum.to_bits() == other.sum.to_bits()
365            && side_layout_equal(
366                &self.positive_spans,
367                &self.positive_buckets,
368                &other.positive_spans,
369                &other.positive_buckets,
370            )
371            && side_layout_equal(
372                &self.negative_spans,
373                &self.negative_buckets,
374                &other.negative_spans,
375                &other.negative_buckets,
376            )
377    }
378
379    /// Formats this histogram using PromQL native histogram sample notation.
380    pub fn promql_string(&self) -> String {
381        let mut parts = vec![format!("count:{}", self.count), format!("sum:{}", self.sum)];
382        if let Some(buckets) = self.all_buckets() {
383            parts.extend(
384                buckets
385                    .into_iter()
386                    .filter(|bucket| bucket.count != 0.0)
387                    .map(|bucket| {
388                        let (left, right) = match bucket.boundary_rule {
389                            BoundaryRule::OpenLeft => ("(", "]"),
390                            BoundaryRule::OpenRight => ("[", ")"),
391                            BoundaryRule::ClosedBoth => ("[", "]"),
392                        };
393                        format!(
394                            "{}{},{}{}:{}",
395                            left, bucket.lower, bucket.upper, right, bucket.count
396                        )
397                    }),
398            );
399        }
400        format!("{{{}}}", parts.join(", "))
401    }
402
403    /// Estimates the population variance using each populated bucket's midpoint.
404    pub fn estimated_stdvar(&self) -> f64 {
405        if self.count == 0.0 {
406            return f64::NAN;
407        }
408        let mean = self.sum / self.count;
409        let Some(buckets) = self.all_buckets() else {
410            return f64::NAN;
411        };
412        buckets
413            .into_iter()
414            .map(|bucket| {
415                let midpoint = self.bucket_midpoint(&bucket);
416                bucket.count * (midpoint - mean).powi(2)
417            })
418            .sum::<f64>()
419            / self.count
420    }
421
422    /// Estimates the population standard deviation from [`Self::estimated_stdvar`].
423    pub fn estimated_stddev(&self) -> f64 {
424        self.estimated_stdvar().sqrt()
425    }
426
427    /// Multiplies every payload value by `factor`.
428    ///
429    /// A negative factor marks the result as a gauge histogram.
430    pub fn scale(mut self, factor: f64) -> Self {
431        self.count *= factor;
432        self.zero_count *= factor;
433        self.sum *= factor;
434        for count in &mut self.positive_buckets {
435            *count *= factor;
436        }
437        for count in &mut self.negative_buckets {
438            *count *= factor;
439        }
440        if factor < 0.0 {
441            self.reset_hint = CounterResetHint::Gauge;
442        }
443        self
444    }
445
446    /// Divides every payload value by `divisor`.
447    ///
448    /// A negative divisor marks the result as a gauge histogram.
449    pub fn divide_by(mut self, divisor: f64) -> Self {
450        self.count /= divisor;
451        self.zero_count /= divisor;
452        self.sum /= divisor;
453        if divisor == 0.0 {
454            self.positive_spans.clear();
455            self.positive_buckets.clear();
456            self.negative_spans.clear();
457            self.negative_buckets.clear();
458        } else {
459            for count in &mut self.positive_buckets {
460                *count /= divisor;
461            }
462            for count in &mut self.negative_buckets {
463                *count /= divisor;
464            }
465        }
466        if divisor < 0.0 {
467            self.reset_hint = CounterResetHint::Gauge;
468        }
469        self
470    }
471
472    fn compact(mut self) -> Option<Self> {
473        let (spans, buckets) = compact_side(&self.positive_spans, &self.positive_buckets)?;
474        self.positive_spans = spans;
475        self.positive_buckets = buckets;
476        let (spans, buckets) = compact_side(&self.negative_spans, &self.negative_buckets)?;
477        self.negative_spans = spans;
478        self.negative_buckets = buckets;
479        Some(self)
480    }
481
482    /// Detects a reset from the explicit hint and histogram payload.
483    pub fn detect_reset(&self, previous: &Self) -> bool {
484        match self.reset_hint {
485            CounterResetHint::CounterReset => return true,
486            CounterResetHint::NotCounterReset => return false,
487            CounterResetHint::Unknown
488            | CounterResetHint::Gauge
489            | CounterResetHint::Unrecognized(_) => {}
490        }
491
492        if self.count < previous.count {
493            return true;
494        }
495
496        match (self.uses_custom_buckets(), previous.uses_custom_buckets()) {
497            (true, true) => {
498                let Some((current, previous)) = reconcile_custom(self, previous) else {
499                    return true;
500                };
501                current.zero_count < previous.zero_count
502                    || current.side_has_reset(true, &previous)
503                    || current.side_has_reset(false, &previous)
504            }
505            (true, false) | (false, true) => true,
506            (false, false) => {
507                if self.schema > previous.schema || self.zero_threshold < previous.zero_threshold {
508                    return true;
509                }
510
511                let mut previous = previous.clone();
512                if self.zero_threshold > previous.zero_threshold {
513                    let Some(expanded) = previous.expanded_zero_threshold(self.zero_threshold)
514                    else {
515                        return true;
516                    };
517                    if expanded != self.zero_threshold
518                        || previous.grow_zero_threshold(self.zero_threshold).is_none()
519                    {
520                        return true;
521                    }
522                }
523                let Some(previous) = previous.copy_to_schema(self.schema) else {
524                    return true;
525                };
526
527                self.zero_count < previous.zero_count
528                    || self.side_has_reset(true, &previous)
529                    || self.side_has_reset(false, &previous)
530            }
531        }
532    }
533
534    fn detect_start_timestamp_reset(
535        &self,
536        previous: &Self,
537        previous_ts: i64,
538        current_ts: i64,
539    ) -> bool {
540        let current_start = self.start_timestamp.unwrap_or_default();
541        if current_start == 0 || current_start >= current_ts || current_start < previous_ts {
542            return false;
543        }
544        if current_start > previous_ts {
545            return true;
546        }
547
548        let previous_start = previous.start_timestamp.unwrap_or_default();
549        previous_start <= previous_ts && previous_start != 0 && previous_start != previous_ts
550    }
551
552    /// Detects a counter reset using start timestamps, hints, and payload values.
553    pub fn detect_counter_reset(&self, previous: &Self, previous_ts: i64, current_ts: i64) -> bool {
554        self.detect_start_timestamp_reset(previous, previous_ts, current_ts)
555            || self.detect_reset(previous)
556    }
557
558    fn all_buckets(&self) -> Option<Vec<Bucket>> {
559        let mut buckets = self.side_buckets(false)?;
560        buckets.reverse();
561        if self.zero_count != 0.0 {
562            buckets.push(Bucket {
563                lower: -self.zero_threshold,
564                upper: self.zero_threshold,
565                count: self.zero_count,
566                boundary_rule: BoundaryRule::ClosedBoth,
567            });
568        }
569        buckets.extend(self.side_buckets(true)?);
570        Some(buckets)
571    }
572
573    fn side_buckets(&self, positive: bool) -> Option<Vec<Bucket>> {
574        let (spans, counts) = if positive {
575            (&self.positive_spans, &self.positive_buckets)
576        } else {
577            (&self.negative_spans, &self.negative_buckets)
578        };
579
580        let mut result = Vec::with_capacity(counts.len());
581        for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(counts) {
582            let upper = get_bound(idx, self.schema, &self.custom_values)?;
583            let lower = get_bound(idx.checked_sub(1)?, self.schema, &self.custom_values)?;
584            if positive {
585                result.push(Bucket {
586                    lower,
587                    upper,
588                    count: *count,
589                    boundary_rule: if self.uses_custom_buckets() && idx == 0 {
590                        BoundaryRule::ClosedBoth
591                    } else {
592                        BoundaryRule::OpenLeft
593                    },
594                });
595            } else {
596                result.push(Bucket {
597                    lower: -upper,
598                    upper: -lower,
599                    count: *count,
600                    boundary_rule: BoundaryRule::OpenRight,
601                });
602            }
603        }
604        Some(result)
605    }
606
607    /// Estimates the value at quantile `q`.
608    ///
609    /// Returns negative or positive infinity outside `[0, 1]`, and `NaN` when
610    /// the quantile cannot be estimated.
611    pub fn quantile(&self, q: f64) -> f64 {
612        if q < 0.0 {
613            return f64::NEG_INFINITY;
614        }
615        if q > 1.0 {
616            return f64::INFINITY;
617        }
618        if self.count == 0.0 || q.is_nan() {
619            return f64::NAN;
620        }
621
622        let Some(mut buckets) = self.all_buckets() else {
623            return f64::NAN;
624        };
625        let rank = q * self.count;
626        let mut count = 0.0;
627        for bucket in &mut buckets {
628            if bucket.count == 0.0 {
629                continue;
630            }
631            count += bucket.count;
632            if count < rank {
633                continue;
634            }
635
636            if !self.uses_custom_buckets() && bucket.lower < 0.0 && bucket.upper > 0.0 {
637                if self.negative_buckets.is_empty() && !self.positive_buckets.is_empty() {
638                    bucket.lower = 0.0;
639                } else if self.positive_buckets.is_empty() && !self.negative_buckets.is_empty() {
640                    bucket.upper = 0.0;
641                }
642            } else if self.uses_custom_buckets() {
643                if bucket.lower == f64::NEG_INFINITY {
644                    if bucket.upper <= 0.0 {
645                        return bucket.upper;
646                    }
647                    bucket.lower = 0.0;
648                } else if bucket.upper == f64::INFINITY {
649                    return bucket.lower;
650                }
651            }
652
653            let rank_in_bucket = rank - (count - bucket.count);
654            let fraction = rank_in_bucket / bucket.count;
655            if self.uses_custom_buckets() || (bucket.lower <= 0.0 && bucket.upper >= 0.0) {
656                return bucket.lower + (bucket.upper - bucket.lower) * fraction;
657            }
658
659            let log_lower = bucket.lower.abs().log2();
660            let log_upper = bucket.upper.abs().log2();
661            if bucket.lower > 0.0 {
662                return 2.0_f64.powf(log_lower + (log_upper - log_lower) * fraction);
663            }
664            return -2.0_f64.powf(log_upper + (log_lower - log_upper) * (1.0 - fraction));
665        }
666
667        f64::NAN
668    }
669
670    /// Estimates the fraction of observations between `lower` and `upper`.
671    pub fn fraction(&self, lower: f64, upper: f64) -> f64 {
672        if self.count == 0.0 || lower.is_nan() || upper.is_nan() {
673            return f64::NAN;
674        }
675        if lower >= upper {
676            return 0.0;
677        }
678
679        let Some(mut buckets) = self.all_buckets() else {
680            return f64::NAN;
681        };
682        let count = if self.sum.is_nan() {
683            buckets.iter().map(|bucket| bucket.count).sum()
684        } else {
685            self.count
686        };
687
688        let mut rank = 0.0;
689        let mut lower_rank = 0.0;
690        let mut upper_rank = 0.0;
691        let mut lower_set = false;
692        let mut upper_set = false;
693
694        for bucket in &mut buckets {
695            let zero_bucket = bucket.lower <= 0.0 && bucket.upper >= 0.0;
696            if zero_bucket {
697                if self.negative_buckets.is_empty() && !self.positive_buckets.is_empty() {
698                    bucket.lower = 0.0;
699                } else if self.positive_buckets.is_empty() && !self.negative_buckets.is_empty() {
700                    bucket.upper = 0.0;
701                }
702            }
703
704            if !lower_set && bucket.lower >= lower {
705                lower_rank = rank;
706                lower_set = true;
707            }
708            if !upper_set && bucket.lower >= upper {
709                upper_rank = rank;
710                upper_set = true;
711            }
712            if lower_set && upper_set {
713                break;
714            }
715            if !lower_set && bucket.lower < lower && bucket.upper > lower {
716                lower_rank = self.interpolate_rank(bucket, rank, lower, zero_bucket);
717                lower_set = true;
718            }
719            if !upper_set && bucket.lower < upper && bucket.upper > upper {
720                upper_rank = self.interpolate_rank(bucket, rank, upper, zero_bucket);
721                upper_set = true;
722            }
723            if lower_set && upper_set {
724                break;
725            }
726            rank += bucket.count;
727        }
728
729        if !lower_set || lower_rank > count {
730            lower_rank = count;
731        }
732        if !upper_set || upper_rank > count {
733            upper_rank = count;
734        }
735
736        (upper_rank - lower_rank) / self.count
737    }
738
739    /// Converts populated buckets to `(boundary rule, lower, upper, count)` strings.
740    pub fn to_prometheus_buckets(&self) -> Option<Vec<(u8, String, String, String)>> {
741        Some(
742            self.all_buckets()?
743                .into_iter()
744                .filter(|bucket| bucket.count != 0.0)
745                .map(|bucket| {
746                    (
747                        bucket.boundary_rule as u8,
748                        format_prometheus_float(bucket.lower),
749                        format_prometheus_float(bucket.upper),
750                        format_prometheus_float(bucket.count),
751                    )
752                })
753                .collect(),
754        )
755    }
756
757    fn interpolate_rank(&self, bucket: &Bucket, rank: f64, value: f64, zero_bucket: bool) -> f64 {
758        if self.uses_custom_buckets() || zero_bucket {
759            if bucket.lower == f64::NEG_INFINITY {
760                return bucket.count;
761            }
762            return rank + bucket.count * (value - bucket.lower) / (bucket.upper - bucket.lower);
763        }
764
765        let log_lower = bucket.lower.abs().log2();
766        let log_upper = bucket.upper.abs().log2();
767        let log_value = value.abs().log2();
768        let fraction = if value > 0.0 {
769            (log_value - log_lower) / (log_upper - log_lower)
770        } else {
771            1.0 - ((log_value - log_upper) / (log_lower - log_upper))
772        };
773        rank + bucket.count * fraction
774    }
775
776    fn bucket_midpoint(&self, bucket: &Bucket) -> f64 {
777        if self.uses_custom_buckets() {
778            return (bucket.lower + bucket.upper) / 2.0;
779        }
780        if bucket.lower <= 0.0 && bucket.upper >= 0.0 {
781            return 0.0;
782        }
783        if bucket.upper < 0.0 {
784            -((bucket.lower.abs() * bucket.upper.abs()).sqrt())
785        } else {
786            (bucket.lower * bucket.upper).sqrt()
787        }
788    }
789
790    fn side_has_reset(&self, positive: bool, previous: &Self) -> bool {
791        let (current_spans, current_buckets, previous_spans, previous_buckets) = if positive {
792            (
793                &self.positive_spans,
794                &self.positive_buckets,
795                &previous.positive_spans,
796                &previous.positive_buckets,
797            )
798        } else {
799            (
800                &self.negative_spans,
801                &self.negative_buckets,
802                &previous.negative_spans,
803                &previous.negative_buckets,
804            )
805        };
806        let Some(current) = side_counts(current_spans, current_buckets) else {
807            return true;
808        };
809        let Some(previous) = side_counts(previous_spans, previous_buckets) else {
810            return true;
811        };
812        previous.keys().chain(current.keys()).any(|idx| {
813            current.get(idx).copied().unwrap_or_default()
814                < previous.get(idx).copied().unwrap_or_default()
815        })
816    }
817
818    fn reconcile(&self, other: &Self) -> Option<(Self, Self)> {
819        match (self.uses_custom_buckets(), other.uses_custom_buckets()) {
820            (true, true) => reconcile_custom(self, other),
821            (false, false) => reconcile_exponential(self, other),
822            _ => None,
823        }
824    }
825}
826
827fn reconcile_exponential(
828    left: &NativeHistogram,
829    right: &NativeHistogram,
830) -> Option<(NativeHistogram, NativeHistogram)> {
831    let schema = left.schema.min(right.schema);
832    let mut left = left.copy_to_schema(schema)?;
833    let mut right = right.copy_to_schema(schema)?;
834    let mut zero_threshold = left.zero_threshold.max(right.zero_threshold);
835    loop {
836        let expanded = left
837            .expanded_zero_threshold(zero_threshold)?
838            .max(right.expanded_zero_threshold(zero_threshold)?);
839        if expanded == zero_threshold {
840            break;
841        }
842        zero_threshold = expanded;
843    }
844    left.grow_zero_threshold(zero_threshold)?;
845    right.grow_zero_threshold(zero_threshold)?;
846    Some((left.compact()?, right.compact()?))
847}
848
849fn reconcile_custom(
850    left: &NativeHistogram,
851    right: &NativeHistogram,
852) -> Option<(NativeHistogram, NativeHistogram)> {
853    let custom_values = if left.custom_values == right.custom_values {
854        left.custom_values.clone()
855    } else {
856        left.custom_values
857            .iter()
858            .copied()
859            .filter(|value| right.custom_values.contains(value))
860            .collect()
861    };
862
863    Some((
864        left.copy_to_custom_values(custom_values.clone())?
865            .compact()?,
866        right.copy_to_custom_values(custom_values)?.compact()?,
867    ))
868}
869
870impl NativeHistogram {
871    fn copy_to_schema(&self, target_schema: i32) -> Option<Self> {
872        if self.uses_custom_buckets()
873            || !(MIN_EXPONENTIAL_SCHEMA..=MAX_EXPONENTIAL_SCHEMA).contains(&target_schema)
874            || target_schema > self.schema
875        {
876            return None;
877        }
878        if target_schema == self.schema {
879            return Some(self.clone());
880        }
881
882        let mut result = self.clone();
883        result.schema = target_schema;
884        (result.positive_spans, result.positive_buckets) = reduce_side(
885            &self.positive_spans,
886            &self.positive_buckets,
887            self.schema,
888            target_schema,
889        )?;
890        (result.negative_spans, result.negative_buckets) = reduce_side(
891            &self.negative_spans,
892            &self.negative_buckets,
893            self.schema,
894            target_schema,
895        )?;
896        Some(result)
897    }
898
899    fn copy_to_custom_values(&self, custom_values: Vec<f64>) -> Option<Self> {
900        if !self.uses_custom_buckets() {
901            return None;
902        }
903        if self.custom_values == custom_values {
904            return Some(self.clone());
905        }
906
907        let mut result = self.clone();
908        result.custom_values = custom_values.clone();
909        result.negative_spans.clear();
910        result.negative_buckets.clear();
911        (result.positive_spans, result.positive_buckets) = map_custom_side(
912            &self.positive_spans,
913            &self.positive_buckets,
914            &self.custom_values,
915            &custom_values,
916        )?;
917        Some(result)
918    }
919
920    fn grow_zero_threshold(&mut self, zero_threshold: f64) -> Option<()> {
921        if self.uses_custom_buckets() || zero_threshold == self.zero_threshold {
922            self.zero_threshold = zero_threshold;
923            return Some(());
924        }
925
926        let (spans, buckets, zero_count) = fold_zero_side(
927            &self.positive_spans,
928            &self.positive_buckets,
929            self.schema,
930            zero_threshold,
931        )?;
932        self.positive_spans = spans;
933        self.positive_buckets = buckets;
934        self.zero_count += zero_count;
935
936        let (spans, buckets, zero_count) = fold_zero_side(
937            &self.negative_spans,
938            &self.negative_buckets,
939            self.schema,
940            zero_threshold,
941        )?;
942        self.negative_spans = spans;
943        self.negative_buckets = buckets;
944        self.zero_count += zero_count;
945        self.zero_threshold = zero_threshold;
946        Some(())
947    }
948
949    fn expanded_zero_threshold(&self, mut zero_threshold: f64) -> Option<f64> {
950        if self.uses_custom_buckets() {
951            return Some(zero_threshold);
952        }
953        zero_threshold = expand_zero_threshold_side(
954            &self.positive_spans,
955            &self.positive_buckets,
956            self.schema,
957            zero_threshold,
958        )?;
959        expand_zero_threshold_side(
960            &self.negative_spans,
961            &self.negative_buckets,
962            self.schema,
963            zero_threshold,
964        )
965    }
966}
967
968fn add_reset_hint(left: CounterResetHint, right: CounterResetHint) -> CounterResetHint {
969    if left == CounterResetHint::Gauge || right == CounterResetHint::Gauge {
970        CounterResetHint::Gauge
971    } else if left == right {
972        left
973    } else {
974        CounterResetHint::Unknown
975    }
976}
977
978fn side_bucket_indices(spans: &[Span]) -> Option<Vec<i32>> {
979    let mut indices = Vec::new();
980    let mut current_index = 0i32;
981    let mut first = true;
982    for (span_index, span) in spans.iter().enumerate() {
983        if span_index > 0 && span.offset < 0 {
984            return None;
985        }
986        if first {
987            current_index = span.offset;
988            first = false;
989        } else {
990            current_index = current_index.checked_add(span.offset)?;
991        }
992        for _ in 0..span.length {
993            indices.push(current_index);
994            current_index = current_index.checked_add(1)?;
995        }
996    }
997    Some(indices)
998}
999
1000fn span_bucket_len(spans: &[Span]) -> Option<usize> {
1001    spans
1002        .iter()
1003        .try_fold(0usize, |sum, span| sum.checked_add(span.length as usize))
1004}
1005
1006fn spans_from_indices_counts(values: Vec<(i32, f64)>) -> Option<(Vec<Span>, Vec<f64>)> {
1007    let mut spans = Vec::<Span>::new();
1008    let mut buckets = Vec::new();
1009    let mut previous_index = None::<i32>;
1010
1011    for (idx, count) in values {
1012        if count == 0.0 {
1013            continue;
1014        }
1015        match (spans.last_mut(), previous_index) {
1016            (Some(span), Some(previous)) if previous.checked_add(1) == Some(idx) => {
1017                span.length = span.length.checked_add(1)?;
1018            }
1019            (_, Some(previous)) => {
1020                spans.push(Span {
1021                    offset: idx.checked_sub(previous)?.checked_sub(1)?,
1022                    length: 1,
1023                });
1024            }
1025            (_, None) => {
1026                spans.push(Span {
1027                    offset: idx,
1028                    length: 1,
1029                });
1030            }
1031        }
1032        buckets.push(count);
1033        previous_index = Some(idx);
1034    }
1035
1036    Some((spans, buckets))
1037}
1038
1039fn side_counts(spans: &[Span], buckets: &[f64]) -> Option<BTreeMap<i32, f64>> {
1040    let mut values = BTreeMap::new();
1041    for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1042        if *count != 0.0 {
1043            values.insert(idx, *count);
1044        }
1045    }
1046    Some(values)
1047}
1048
1049fn side_layout_equal(
1050    left_spans: &[Span],
1051    left_buckets: &[f64],
1052    right_spans: &[Span],
1053    right_buckets: &[f64],
1054) -> bool {
1055    if span_bucket_len(left_spans) != Some(left_buckets.len())
1056        || span_bucket_len(right_spans) != Some(right_buckets.len())
1057    {
1058        return false;
1059    }
1060    let Some(left_indices) = side_bucket_indices(left_spans) else {
1061        return false;
1062    };
1063    let Some(right_indices) = side_bucket_indices(right_spans) else {
1064        return false;
1065    };
1066    left_indices == right_indices
1067        && left_buckets
1068            .iter()
1069            .zip(right_buckets)
1070            .all(|(left, right)| left.to_bits() == right.to_bits())
1071}
1072
1073fn merge_side(
1074    left_spans: &[Span],
1075    left_buckets: &[f64],
1076    right_spans: &[Span],
1077    right_buckets: &[f64],
1078    op: impl Fn(f64, f64) -> f64,
1079) -> Option<(Vec<Span>, Vec<f64>)> {
1080    let left = side_counts(left_spans, left_buckets)?;
1081    let right = side_counts(right_spans, right_buckets)?;
1082    let mut values = BTreeMap::new();
1083    for idx in left.keys().chain(right.keys()) {
1084        values.insert(
1085            *idx,
1086            op(
1087                left.get(idx).copied().unwrap_or_default(),
1088                right.get(idx).copied().unwrap_or_default(),
1089            ),
1090        );
1091    }
1092    spans_from_indices_counts(values.into_iter().collect())
1093}
1094
1095fn reduce_side(
1096    spans: &[Span],
1097    buckets: &[f64],
1098    schema: i32,
1099    target_schema: i32,
1100) -> Option<(Vec<Span>, Vec<f64>)> {
1101    let factor = 1_i32.checked_shl((schema - target_schema) as u32)?;
1102    let mut values = std::collections::BTreeMap::<i32, f64>::new();
1103    for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1104        let target_idx = ceil_div(idx, factor);
1105        *values.entry(target_idx).or_default() += *count;
1106    }
1107    spans_from_indices_counts(values.into_iter().collect())
1108}
1109
1110fn compact_side(spans: &[Span], buckets: &[f64]) -> Option<(Vec<Span>, Vec<f64>)> {
1111    let indices = side_bucket_indices(spans)?;
1112    spans_from_indices_counts(indices.into_iter().zip(buckets.iter().copied()).collect())
1113}
1114
1115fn map_custom_side(
1116    spans: &[Span],
1117    buckets: &[f64],
1118    old_values: &[f64],
1119    new_values: &[f64],
1120) -> Option<(Vec<Span>, Vec<f64>)> {
1121    let mut values = std::collections::BTreeMap::<i32, f64>::new();
1122    for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1123        let upper = get_bound(idx, CUSTOM_BUCKETS_SCHEMA, old_values)?;
1124        let target_idx = new_values
1125            .iter()
1126            .position(|value| *value >= upper)
1127            .unwrap_or(new_values.len()) as i32;
1128        *values.entry(target_idx).or_default() += *count;
1129    }
1130    spans_from_indices_counts(values.into_iter().collect())
1131}
1132
1133fn fold_zero_side(
1134    spans: &[Span],
1135    buckets: &[f64],
1136    schema: i32,
1137    zero_threshold: f64,
1138) -> Option<(Vec<Span>, Vec<f64>, f64)> {
1139    let mut kept = Vec::new();
1140    let mut zero_count = 0.0;
1141    for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1142        if get_bound(idx, schema, &[])? <= zero_threshold {
1143            zero_count += *count;
1144        } else {
1145            kept.push((idx, *count));
1146        }
1147    }
1148    let (spans, buckets) = spans_from_indices_counts(kept)?;
1149    Some((spans, buckets, zero_count))
1150}
1151
1152fn expand_zero_threshold_side(
1153    spans: &[Span],
1154    buckets: &[f64],
1155    schema: i32,
1156    mut zero_threshold: f64,
1157) -> Option<f64> {
1158    for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1159        if *count == 0.0 {
1160            continue;
1161        }
1162        let lower = get_bound(idx.checked_sub(1)?, schema, &[])?;
1163        let upper = get_bound(idx, schema, &[])?;
1164        if lower < zero_threshold && zero_threshold < upper {
1165            zero_threshold = upper;
1166        }
1167    }
1168    Some(zero_threshold)
1169}
1170
1171fn ceil_div(value: i32, divisor: i32) -> i32 {
1172    value.div_euclid(divisor) + i32::from(value.rem_euclid(divisor) != 0)
1173}
1174
1175/// Returns the index of the exponential bucket containing positive infinity.
1176pub fn exponential_overflow_bucket_index(schema: i32) -> Option<i32> {
1177    if !(MIN_EXPONENTIAL_SCHEMA..=MAX_EXPONENTIAL_SCHEMA).contains(&schema) {
1178        return None;
1179    }
1180    let last_finite = if schema >= 0 {
1181        1024_i64.checked_shl(schema as u32)?
1182    } else {
1183        1024_i64.checked_shr((-schema) as u32)?
1184    };
1185    i32::try_from(last_finite.checked_add(1)?).ok()
1186}
1187
1188fn get_bound(idx: i32, schema: i32, custom_values: &[f64]) -> Option<f64> {
1189    if schema == CUSTOM_BUCKETS_SCHEMA {
1190        return match idx {
1191            -1 => Some(f64::NEG_INFINITY),
1192            idx if idx == custom_values.len() as i32 => Some(f64::INFINITY),
1193            idx if idx >= 0 && (idx as usize) < custom_values.len() => {
1194                Some(custom_values[idx as usize])
1195            }
1196            _ => None,
1197        };
1198    }
1199
1200    let overflow_index = exponential_overflow_bucket_index(schema)?;
1201    if idx > overflow_index {
1202        return None;
1203    }
1204    if idx == overflow_index {
1205        return Some(f64::INFINITY);
1206    }
1207    if idx == overflow_index - 1 {
1208        return Some(f64::MAX);
1209    }
1210    if schema < 0 {
1211        let exponent = i64::from(idx).checked_shl((-schema) as u32)?;
1212        let Ok(exponent) = i32::try_from(exponent) else {
1213            return Some(0.0);
1214        };
1215        return Some(2.0_f64.powi(exponent));
1216    }
1217    let exponent = idx as f64 / (1u32 << schema) as f64;
1218    Some(2.0_f64.powf(exponent))
1219}
1220
1221/// Returns the Arrow type for a complete native histogram value.
1222pub fn native_histogram_arrow_type() -> ArrowDataType {
1223    native_histogram_value_type().as_arrow_type()
1224}
1225
1226fn struct_child<'a>(array: &'a StructArray, name: &str) -> DfResult<&'a ArrayRef> {
1227    let index = array
1228        .fields()
1229        .iter()
1230        .position(|field| field.name() == name)
1231        .ok_or_else(|| {
1232            DataFusionError::Execution(format!("native histogram missing field {name}"))
1233        })?;
1234    Ok(array.column(index))
1235}
1236
1237fn primitive_child<'a, T>(array: &'a StructArray, name: &str) -> DfResult<&'a PrimitiveArray<T>>
1238where
1239    T: ArrowPrimitiveType,
1240{
1241    let child = struct_child(array, name)?;
1242    child
1243        .as_any()
1244        .downcast_ref::<PrimitiveArray<T>>()
1245        .ok_or_else(|| {
1246            DataFusionError::Execution(format!(
1247                "native histogram field {name} has invalid type {}",
1248                child.data_type()
1249            ))
1250        })
1251}
1252
1253fn list_child<'a>(array: &'a StructArray, name: &str) -> DfResult<&'a ListArray> {
1254    let child = struct_child(array, name)?;
1255    child.as_any().downcast_ref::<ListArray>().ok_or_else(|| {
1256        DataFusionError::Execution(format!(
1257            "native histogram field {name} has invalid type {}",
1258            child.data_type()
1259        ))
1260    })
1261}
1262
1263fn required_primitive<T>(array: &StructArray, name: &str, row: usize) -> DfResult<T::Native>
1264where
1265    T: ArrowPrimitiveType,
1266{
1267    let values = primitive_child::<T>(array, name)?;
1268    if values.is_null(row) {
1269        return Err(DataFusionError::Execution(format!(
1270            "native histogram field {name} is null"
1271        )));
1272    }
1273    Ok(values.value(row))
1274}
1275
1276fn optional_primitive<T>(array: &StructArray, name: &str, row: usize) -> DfResult<Option<T::Native>>
1277where
1278    T: ArrowPrimitiveType,
1279{
1280    let values = primitive_child::<T>(array, name)?;
1281    Ok((!values.is_null(row)).then(|| values.value(row)))
1282}
1283
1284fn list_values<T>(array: &StructArray, name: &str, row: usize) -> DfResult<Vec<T::Native>>
1285where
1286    T: ArrowPrimitiveType,
1287{
1288    let list = list_child(array, name)?;
1289    if list.is_null(row) {
1290        return Ok(Vec::new());
1291    }
1292
1293    let values = list.value(row);
1294    let values = values
1295        .as_any()
1296        .downcast_ref::<PrimitiveArray<T>>()
1297        .ok_or_else(|| {
1298            DataFusionError::Execution(format!(
1299                "native histogram list field {name} has invalid value type {}",
1300                values.data_type()
1301            ))
1302        })?;
1303
1304    values
1305        .iter()
1306        .map(|value| {
1307            value.ok_or_else(|| {
1308                DataFusionError::Execution(format!(
1309                    "native histogram list field {name} contains null"
1310                ))
1311            })
1312        })
1313        .collect()
1314}
1315
1316fn read_spans(offsets: Vec<i32>, lengths: Vec<u32>, name: &str) -> DfResult<Vec<Span>> {
1317    if offsets.len() != lengths.len() {
1318        return Err(DataFusionError::Execution(format!(
1319            "native histogram {name} span offsets and lengths mismatch: {} vs {}",
1320            offsets.len(),
1321            lengths.len()
1322        )));
1323    }
1324    Ok(offsets
1325        .into_iter()
1326        .zip(lengths)
1327        .map(|(offset, length)| Span { offset, length })
1328        .collect())
1329}
1330
1331fn check_span_bucket_count(spans: &[Span], buckets: usize, name: &str) -> DfResult<()> {
1332    let span_len = span_bucket_len(spans).ok_or_else(|| {
1333        DataFusionError::Execution(format!("native histogram {name} spans overflow"))
1334    })?;
1335    if span_len != buckets {
1336        return Err(DataFusionError::Execution(format!(
1337            "native histogram {name} spans describe {span_len} buckets, found {buckets}"
1338        )));
1339    }
1340    Ok(())
1341}
1342
1343/// Decodes one native histogram row into the query-time [`NativeHistogram`] model.
1344///
1345/// A null struct returns `Ok(None)`. Integer payloads are converted to absolute
1346/// `f64` counts, and malformed child fields or span layouts return an error.
1347pub fn read_histogram(array: &StructArray, row: usize) -> DfResult<Option<NativeHistogram>> {
1348    if array.is_null(row) {
1349        return Ok(None);
1350    }
1351
1352    let schema = required_primitive::<Int32Type>(array, SCHEMA_FIELD, row)?;
1353    let positive_spans = read_spans(
1354        list_values::<Int32Type>(array, POSITIVE_SPAN_OFFSETS_FIELD, row)?,
1355        list_values::<UInt32Type>(array, POSITIVE_SPAN_LENGTHS_FIELD, row)?,
1356        "positive",
1357    )?;
1358    let negative_spans = read_spans(
1359        list_values::<Int32Type>(array, NEGATIVE_SPAN_OFFSETS_FIELD, row)?,
1360        list_values::<UInt32Type>(array, NEGATIVE_SPAN_LENGTHS_FIELD, row)?,
1361        "negative",
1362    )?;
1363
1364    let (count, zero_count, positive_buckets, negative_buckets) =
1365        if let Some(count) = optional_primitive::<Float64Type>(array, COUNT_F64_FIELD, row)? {
1366            (
1367                count,
1368                optional_primitive::<Float64Type>(array, ZERO_COUNT_F64_FIELD, row)?
1369                    .unwrap_or_default(),
1370                list_values::<Float64Type>(array, POSITIVE_BUCKETS_F64_FIELD, row)?,
1371                list_values::<Float64Type>(array, NEGATIVE_BUCKETS_F64_FIELD, row)?,
1372            )
1373        } else {
1374            (
1375                required_primitive::<UInt64Type>(array, COUNT_U64_FIELD, row)? as f64,
1376                optional_primitive::<UInt64Type>(array, ZERO_COUNT_U64_FIELD, row)?
1377                    .unwrap_or_default() as f64,
1378                list_values::<Int64Type>(array, POSITIVE_BUCKETS_I64_FIELD, row)?
1379                    .into_iter()
1380                    .map(|value| value as f64)
1381                    .collect(),
1382                list_values::<Int64Type>(array, NEGATIVE_BUCKETS_I64_FIELD, row)?
1383                    .into_iter()
1384                    .map(|value| value as f64)
1385                    .collect(),
1386            )
1387        };
1388
1389    check_span_bucket_count(&positive_spans, positive_buckets.len(), "positive")?;
1390    check_span_bucket_count(&negative_spans, negative_buckets.len(), "negative")?;
1391
1392    Ok(Some(NativeHistogram {
1393        schema,
1394        zero_threshold: required_primitive::<Float64Type>(array, ZERO_THRESHOLD_FIELD, row)?,
1395        sum: required_primitive::<Float64Type>(array, SUM_FIELD, row)?,
1396        reset_hint: required_primitive::<Int32Type>(array, RESET_HINT_FIELD, row)?.into(),
1397        start_timestamp: optional_primitive::<TimestampMillisecondType>(
1398            array,
1399            START_TIMESTAMP_FIELD,
1400            row,
1401        )?,
1402        custom_values: list_values::<Float64Type>(array, CUSTOM_VALUES_FIELD, row)?,
1403        positive_spans,
1404        negative_spans,
1405        count,
1406        zero_count,
1407        positive_buckets,
1408        negative_buckets,
1409    }))
1410}
1411
1412fn list_opt<T>(values: Vec<T>) -> Option<Vec<Option<T>>> {
1413    Some(values.into_iter().map(Some).collect())
1414}
1415
1416/// Builds a native histogram Struct array from query-time values.
1417///
1418/// The result uses the canonical `f64` payload fields. Integer count and bucket
1419/// fields remain empty because [`NativeHistogram`] no longer tracks the source
1420/// payload family after decoding.
1421pub fn build_histogram_array(values: &[Option<NativeHistogram>]) -> ArrayRef {
1422    let mut schemas = Vec::with_capacity(values.len());
1423    let mut zero_thresholds = Vec::with_capacity(values.len());
1424    let mut sums = Vec::with_capacity(values.len());
1425    let mut reset_hints = Vec::with_capacity(values.len());
1426    let mut start_timestamps = Vec::with_capacity(values.len());
1427    let mut custom_values = Vec::with_capacity(values.len());
1428    let mut positive_span_offsets = Vec::with_capacity(values.len());
1429    let mut positive_span_lengths = Vec::with_capacity(values.len());
1430    let mut negative_span_offsets = Vec::with_capacity(values.len());
1431    let mut negative_span_lengths = Vec::with_capacity(values.len());
1432    let mut count_u64 = Vec::<Option<u64>>::with_capacity(values.len());
1433    let mut zero_count_u64 = Vec::<Option<u64>>::with_capacity(values.len());
1434    let mut positive_buckets_i64 = Vec::with_capacity(values.len());
1435    let mut negative_buckets_i64 = Vec::with_capacity(values.len());
1436    let mut count_f64 = Vec::with_capacity(values.len());
1437    let mut zero_count_f64 = Vec::with_capacity(values.len());
1438    let mut positive_buckets_f64 = Vec::with_capacity(values.len());
1439    let mut negative_buckets_f64 = Vec::with_capacity(values.len());
1440    let mut validity = Vec::with_capacity(values.len());
1441
1442    for value in values {
1443        validity.push(value.is_some());
1444        if let Some(histogram) = value {
1445            schemas.push(Some(histogram.schema));
1446            zero_thresholds.push(Some(histogram.zero_threshold));
1447            sums.push(Some(histogram.sum));
1448            reset_hints.push(Some(i32::from(histogram.reset_hint)));
1449            start_timestamps.push(histogram.start_timestamp);
1450            custom_values.push(list_opt(histogram.custom_values.clone()));
1451            positive_span_offsets.push(list_opt(
1452                histogram
1453                    .positive_spans
1454                    .iter()
1455                    .map(|span| span.offset)
1456                    .collect(),
1457            ));
1458            positive_span_lengths.push(list_opt(
1459                histogram
1460                    .positive_spans
1461                    .iter()
1462                    .map(|span| span.length)
1463                    .collect(),
1464            ));
1465            negative_span_offsets.push(list_opt(
1466                histogram
1467                    .negative_spans
1468                    .iter()
1469                    .map(|span| span.offset)
1470                    .collect(),
1471            ));
1472            negative_span_lengths.push(list_opt(
1473                histogram
1474                    .negative_spans
1475                    .iter()
1476                    .map(|span| span.length)
1477                    .collect(),
1478            ));
1479            count_u64.push(None);
1480            zero_count_u64.push(None);
1481            positive_buckets_i64.push(list_opt(Vec::<i64>::new()));
1482            negative_buckets_i64.push(list_opt(Vec::<i64>::new()));
1483            count_f64.push(Some(histogram.count));
1484            zero_count_f64.push(Some(histogram.zero_count));
1485            positive_buckets_f64.push(list_opt(histogram.positive_buckets.clone()));
1486            negative_buckets_f64.push(list_opt(histogram.negative_buckets.clone()));
1487        } else {
1488            schemas.push(None);
1489            zero_thresholds.push(None);
1490            sums.push(None);
1491            reset_hints.push(None);
1492            start_timestamps.push(None);
1493            custom_values.push(None);
1494            positive_span_offsets.push(None);
1495            positive_span_lengths.push(None);
1496            negative_span_offsets.push(None);
1497            negative_span_lengths.push(None);
1498            count_u64.push(None);
1499            zero_count_u64.push(None);
1500            positive_buckets_i64.push(None);
1501            negative_buckets_i64.push(None);
1502            count_f64.push(None);
1503            zero_count_f64.push(None);
1504            positive_buckets_f64.push(None);
1505            negative_buckets_f64.push(None);
1506        }
1507    }
1508
1509    let named_arrays: Vec<(&str, ArrayRef)> = vec![
1510        (SCHEMA_FIELD, Arc::new(Int32Array::from(schemas))),
1511        (
1512            ZERO_THRESHOLD_FIELD,
1513            Arc::new(Float64Array::from(zero_thresholds)),
1514        ),
1515        (SUM_FIELD, Arc::new(Float64Array::from(sums))),
1516        (RESET_HINT_FIELD, Arc::new(Int32Array::from(reset_hints))),
1517        (
1518            START_TIMESTAMP_FIELD,
1519            Arc::new(TimestampMillisecondArray::from_iter(start_timestamps)),
1520        ),
1521        (
1522            CUSTOM_VALUES_FIELD,
1523            Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(
1524                custom_values,
1525            )),
1526        ),
1527        (
1528            POSITIVE_SPAN_OFFSETS_FIELD,
1529            Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
1530                positive_span_offsets,
1531            )),
1532        ),
1533        (
1534            POSITIVE_SPAN_LENGTHS_FIELD,
1535            Arc::new(ListArray::from_iter_primitive::<UInt32Type, _, _>(
1536                positive_span_lengths,
1537            )),
1538        ),
1539        (
1540            NEGATIVE_SPAN_OFFSETS_FIELD,
1541            Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
1542                negative_span_offsets,
1543            )),
1544        ),
1545        (
1546            NEGATIVE_SPAN_LENGTHS_FIELD,
1547            Arc::new(ListArray::from_iter_primitive::<UInt32Type, _, _>(
1548                negative_span_lengths,
1549            )),
1550        ),
1551        (COUNT_U64_FIELD, Arc::new(UInt64Array::from(count_u64))),
1552        (
1553            ZERO_COUNT_U64_FIELD,
1554            Arc::new(UInt64Array::from(zero_count_u64)),
1555        ),
1556        (
1557            POSITIVE_BUCKETS_I64_FIELD,
1558            Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(
1559                positive_buckets_i64,
1560            )),
1561        ),
1562        (
1563            NEGATIVE_BUCKETS_I64_FIELD,
1564            Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(
1565                negative_buckets_i64,
1566            )),
1567        ),
1568        (COUNT_F64_FIELD, Arc::new(Float64Array::from(count_f64))),
1569        (
1570            ZERO_COUNT_F64_FIELD,
1571            Arc::new(Float64Array::from(zero_count_f64)),
1572        ),
1573        (
1574            POSITIVE_BUCKETS_F64_FIELD,
1575            Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(
1576                positive_buckets_f64,
1577            )),
1578        ),
1579        (
1580            NEGATIVE_BUCKETS_F64_FIELD,
1581            Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(
1582                negative_buckets_f64,
1583            )),
1584        ),
1585    ];
1586    let (fields, arrays): (Vec<_>, Vec<_>) = named_arrays
1587        .into_iter()
1588        .map(|(name, array)| (Field::new(name, array.data_type().clone(), true), array))
1589        .unzip();
1590
1591    Arc::new(StructArray::new(
1592        fields.into(),
1593        arrays,
1594        Some(NullBuffer::from(validity)),
1595    ))
1596}
1597
1598#[cfg(test)]
1599mod tests {
1600    use super::*;
1601
1602    fn histogram(positive_spans: Vec<Span>, positive_buckets: Vec<f64>) -> NativeHistogram {
1603        let count = positive_buckets.iter().sum();
1604        NativeHistogram {
1605            schema: 0,
1606            zero_threshold: 0.0,
1607            sum: count,
1608            reset_hint: CounterResetHint::CounterReset,
1609            start_timestamp: None,
1610            custom_values: Vec::new(),
1611            positive_spans,
1612            negative_spans: Vec::new(),
1613            count,
1614            zero_count: 0.0,
1615            positive_buckets,
1616            negative_buckets: Vec::new(),
1617        }
1618    }
1619
1620    #[test]
1621    fn span_rebuild_checks_offsets_and_preserves_empty_input() {
1622        assert_eq!(
1623            spans_from_indices_counts(Vec::new()),
1624            Some((Vec::new(), Vec::new()))
1625        );
1626        assert_eq!(
1627            spans_from_indices_counts(vec![(2, 1.0), (3, 2.0), (5, 3.0)]),
1628            Some((
1629                vec![
1630                    Span {
1631                        offset: 2,
1632                        length: 2,
1633                    },
1634                    Span {
1635                        offset: 1,
1636                        length: 1,
1637                    },
1638                ],
1639                vec![1.0, 2.0, 3.0],
1640            ))
1641        );
1642        assert_eq!(
1643            spans_from_indices_counts(vec![(i32::MIN, 1.0), (i32::MAX, 2.0)]),
1644            None
1645        );
1646    }
1647
1648    #[test]
1649    fn arrow_round_trip_preserves_unrecognized_reset_hint() {
1650        let mut expected = histogram(
1651            vec![Span {
1652                offset: 0,
1653                length: 1,
1654            }],
1655            vec![1.0],
1656        );
1657        expected.reset_hint = CounterResetHint::Unrecognized(42);
1658
1659        let array = build_histogram_array(&[Some(expected.clone())]);
1660        assert_eq!(array.data_type(), &native_histogram_arrow_type());
1661        let array = array.as_any().downcast_ref::<StructArray>().unwrap();
1662
1663        assert_eq!(read_histogram(array, 0).unwrap(), Some(expected));
1664        assert!(
1665            primitive_child::<UInt64Type>(array, COUNT_U64_FIELD)
1666                .unwrap()
1667                .is_null(0)
1668        );
1669        assert_eq!(
1670            primitive_child::<Float64Type>(array, COUNT_F64_FIELD)
1671                .unwrap()
1672                .value(0),
1673            1.0
1674        );
1675    }
1676
1677    #[test]
1678    fn add_uses_sparse_bucket_union() {
1679        let left = histogram(
1680            vec![Span {
1681                offset: 0,
1682                length: 1,
1683            }],
1684            vec![1.0],
1685        );
1686        let right = histogram(
1687            vec![Span {
1688                offset: 2,
1689                length: 1,
1690            }],
1691            vec![2.0],
1692        );
1693
1694        let result = left.add(&right).unwrap();
1695        assert_eq!(
1696            result.positive_spans,
1697            vec![
1698                Span {
1699                    offset: 0,
1700                    length: 1,
1701                },
1702                Span {
1703                    offset: 1,
1704                    length: 1,
1705                },
1706            ]
1707        );
1708        assert_eq!(result.positive_buckets, vec![1.0, 2.0]);
1709    }
1710
1711    #[test]
1712    fn add_combines_reset_hints_for_compatible_empty_payloads() {
1713        let mut empty = histogram(Vec::new(), Vec::new());
1714        empty.reset_hint = CounterResetHint::Gauge;
1715        let mut populated = histogram(
1716            vec![Span {
1717                offset: 0,
1718                length: 1,
1719            }],
1720            vec![1.0],
1721        );
1722        populated.reset_hint = CounterResetHint::NotCounterReset;
1723
1724        assert_eq!(
1725            empty.add(&populated).unwrap().reset_hint,
1726            CounterResetHint::Gauge
1727        );
1728        assert_eq!(
1729            populated.add(&empty).unwrap().reset_hint,
1730            CounterResetHint::Gauge
1731        );
1732    }
1733
1734    #[test]
1735    fn empty_arithmetic_rejects_incompatible_schemas() {
1736        let exponential = histogram(Vec::new(), Vec::new());
1737        let mut custom = histogram(
1738            vec![Span {
1739                offset: 0,
1740                length: 1,
1741            }],
1742            vec![2.0],
1743        );
1744        custom.schema = CUSTOM_BUCKETS_SCHEMA;
1745        custom.custom_values = vec![1.0];
1746
1747        assert!(exponential.add(&custom).is_none());
1748        assert!(custom.add(&exponential).is_none());
1749        assert!(exponential.sub(&custom).is_none());
1750        assert!(custom.sub(&exponential).is_none());
1751    }
1752
1753    #[test]
1754    fn empty_custom_histogram_still_reconciles_bounds() {
1755        let mut empty = histogram(Vec::new(), Vec::new());
1756        empty.schema = CUSTOM_BUCKETS_SCHEMA;
1757        empty.custom_values = vec![1.0];
1758        let mut populated = histogram(
1759            vec![Span {
1760                offset: 0,
1761                length: 1,
1762            }],
1763            vec![2.0],
1764        );
1765        populated.schema = CUSTOM_BUCKETS_SCHEMA;
1766        populated.custom_values = vec![2.0];
1767
1768        let result = empty.add(&populated).unwrap();
1769        assert!(result.custom_values.is_empty());
1770        assert_eq!(result.positive_buckets, vec![2.0]);
1771    }
1772
1773    #[test]
1774    fn zero_threshold_expands_to_populated_bucket_boundary() {
1775        let left = NativeHistogram {
1776            zero_threshold: 0.5,
1777            ..histogram(
1778                vec![Span {
1779                    offset: 0,
1780                    length: 1,
1781                }],
1782                vec![1.0],
1783            )
1784        };
1785        let right = NativeHistogram {
1786            zero_threshold: 0.75,
1787            ..histogram(
1788                vec![Span {
1789                    offset: 1,
1790                    length: 1,
1791                }],
1792                vec![1.0],
1793            )
1794        };
1795
1796        let result = left.add(&right).unwrap();
1797        assert_eq!(result.zero_threshold, 1.0);
1798        assert_eq!(result.zero_count, 1.0);
1799        assert_eq!(result.positive_buckets, vec![1.0]);
1800    }
1801
1802    #[test]
1803    fn custom_reconciliation_without_shared_bounds_uses_overflow_bucket() {
1804        let mut left = histogram(
1805            vec![Span {
1806                offset: 0,
1807                length: 1,
1808            }],
1809            vec![1.0],
1810        );
1811        left.schema = CUSTOM_BUCKETS_SCHEMA;
1812        left.custom_values = vec![1.0];
1813
1814        let mut right = histogram(
1815            vec![Span {
1816                offset: 0,
1817                length: 1,
1818            }],
1819            vec![2.0],
1820        );
1821        right.schema = CUSTOM_BUCKETS_SCHEMA;
1822        right.custom_values = vec![2.0];
1823
1824        let result = left.add(&right).unwrap();
1825        assert!(result.custom_values.is_empty());
1826        assert_eq!(
1827            result.positive_spans,
1828            vec![Span {
1829                offset: 0,
1830                length: 1,
1831            }]
1832        );
1833        assert_eq!(result.positive_buckets, vec![3.0]);
1834    }
1835
1836    #[test]
1837    fn promql_eq_ignores_metadata_but_compares_sparse_layout() {
1838        let mut left = histogram(
1839            vec![Span {
1840                offset: 0,
1841                length: 2,
1842            }],
1843            vec![1.0, 0.0],
1844        );
1845        left.reset_hint = CounterResetHint::CounterReset;
1846        left.start_timestamp = Some(1000);
1847
1848        let mut right = histogram(
1849            vec![Span {
1850                offset: 0,
1851                length: 1,
1852            }],
1853            vec![1.0],
1854        );
1855        right.reset_hint = CounterResetHint::NotCounterReset;
1856        right.start_timestamp = Some(2000);
1857
1858        assert!(!left.promql_eq(&right));
1859
1860        right.positive_spans = vec![
1861            Span {
1862                offset: 0,
1863                length: 0,
1864            },
1865            Span {
1866                offset: 0,
1867                length: 1,
1868            },
1869            Span {
1870                offset: 42,
1871                length: 0,
1872            },
1873        ];
1874        left.positive_spans = vec![Span {
1875            offset: 0,
1876            length: 1,
1877        }];
1878        left.positive_buckets = vec![1.0];
1879        assert!(left.promql_eq(&right));
1880    }
1881
1882    #[test]
1883    fn promql_eq_compares_payload_floats_by_bits() {
1884        let nan = f64::from_bits(0x7ff8_0000_0000_0001);
1885        let other_nan = f64::from_bits(0x7ff8_0000_0000_0002);
1886        let mut left = histogram(
1887            vec![Span {
1888                offset: 0,
1889                length: 1,
1890            }],
1891            vec![nan],
1892        );
1893        left.count = nan;
1894        left.zero_count = nan;
1895        left.sum = nan;
1896        let right = left.clone();
1897
1898        assert!(left.promql_eq(&right));
1899
1900        for changed in [
1901            NativeHistogram {
1902                count: other_nan,
1903                ..right.clone()
1904            },
1905            NativeHistogram {
1906                zero_count: other_nan,
1907                ..right.clone()
1908            },
1909            NativeHistogram {
1910                sum: other_nan,
1911                ..right.clone()
1912            },
1913            NativeHistogram {
1914                positive_buckets: vec![other_nan],
1915                ..right.clone()
1916            },
1917        ] {
1918            assert!(!left.promql_eq(&changed));
1919        }
1920
1921        let undecodable = histogram(
1922            vec![Span {
1923                offset: i32::MAX,
1924                length: 1,
1925            }],
1926            vec![1.0],
1927        );
1928        assert!(!undecodable.promql_eq(&undecodable.clone()));
1929
1930        let mut mismatched = histogram(
1931            vec![Span {
1932                offset: 0,
1933                length: 2,
1934            }],
1935            vec![1.0],
1936        );
1937        mismatched.count = 1.0;
1938        mismatched.sum = 1.0;
1939        assert!(!mismatched.promql_eq(&mismatched.clone()));
1940
1941        let invalid_offset = histogram(
1942            vec![
1943                Span {
1944                    offset: 0,
1945                    length: 1,
1946                },
1947                Span {
1948                    offset: -1,
1949                    length: 1,
1950                },
1951            ],
1952            vec![1.0, 1.0],
1953        );
1954        assert!(!invalid_offset.promql_eq(&invalid_offset.clone()));
1955    }
1956
1957    #[test]
1958    fn detect_reset_preserves_layout_direction() {
1959        let mut previous = histogram(
1960            vec![Span {
1961                offset: 1,
1962                length: 1,
1963            }],
1964            vec![1.0],
1965        );
1966        previous.reset_hint = CounterResetHint::Unknown;
1967
1968        let mut higher_resolution = previous.clone();
1969        higher_resolution.schema = 1;
1970        higher_resolution.positive_spans[0].offset = 2;
1971        assert!(higher_resolution.detect_reset(&previous));
1972
1973        let mut lower_resolution = previous.clone();
1974        lower_resolution.schema = 0;
1975        lower_resolution.positive_spans[0].offset = 1;
1976        let mut previous_higher_resolution = higher_resolution;
1977        previous_higher_resolution.reset_hint = CounterResetHint::Unknown;
1978        assert!(!lower_resolution.detect_reset(&previous_higher_resolution));
1979
1980        let mut smaller_zero_threshold = previous.clone();
1981        smaller_zero_threshold.zero_threshold = 0.5;
1982        previous.zero_threshold = 1.0;
1983        assert!(smaller_zero_threshold.detect_reset(&previous));
1984
1985        let mut previous = histogram(
1986            vec![Span {
1987                offset: 0,
1988                length: 1,
1989            }],
1990            vec![1.0],
1991        );
1992        previous.reset_hint = CounterResetHint::Unknown;
1993        let mut split_bucket = histogram(Vec::new(), Vec::new());
1994        split_bucket.reset_hint = CounterResetHint::Unknown;
1995        split_bucket.count = 1.0;
1996        split_bucket.zero_count = 1.0;
1997        split_bucket.zero_threshold = 0.75;
1998        assert!(split_bucket.detect_reset(&previous));
1999
2000        split_bucket.zero_threshold = 1.0;
2001        assert!(!split_bucket.detect_reset(&previous));
2002    }
2003
2004    #[test]
2005    fn extreme_bucket_index_returns_none_instead_of_overflowing() {
2006        assert_eq!(ceil_div(i32::MIN, 2), i32::MIN / 2);
2007
2008        let mut histogram = histogram(
2009            vec![Span {
2010                offset: i32::MIN,
2011                length: 1,
2012            }],
2013            vec![1.0],
2014        );
2015        assert!(histogram.side_buckets(true).is_none());
2016
2017        histogram.schema = 1;
2018        let reduced = histogram.copy_to_schema(0).unwrap();
2019        assert_eq!(reduced.positive_spans[0].offset, i32::MIN / 2);
2020    }
2021
2022    #[test]
2023    fn exponential_bucket_bounds_stop_after_overflow_bucket() {
2024        for schema in [-4, 0, 8] {
2025            let overflow = exponential_overflow_bucket_index(schema).unwrap();
2026            assert_eq!(get_bound(overflow - 1, schema, &[]), Some(f64::MAX));
2027            assert_eq!(get_bound(overflow, schema, &[]), Some(f64::INFINITY));
2028            assert_eq!(get_bound(overflow + 1, schema, &[]), None);
2029        }
2030        assert_eq!(exponential_overflow_bucket_index(-5), None);
2031        assert_eq!(exponential_overflow_bucket_index(9), None);
2032        assert_eq!(get_bound(i32::MIN, -4, &[]), Some(0.0));
2033    }
2034
2035    #[test]
2036    fn custom_bucket_midpoints_preserve_infinite_bounds() {
2037        let mut histogram = histogram(Vec::new(), Vec::new());
2038        histogram.schema = CUSTOM_BUCKETS_SCHEMA;
2039
2040        assert_eq!(
2041            histogram.bucket_midpoint(&Bucket {
2042                lower: f64::NEG_INFINITY,
2043                upper: -1.0,
2044                count: 1.0,
2045                boundary_rule: BoundaryRule::OpenLeft,
2046            }),
2047            f64::NEG_INFINITY
2048        );
2049        assert_eq!(
2050            histogram.bucket_midpoint(&Bucket {
2051                lower: 1.0,
2052                upper: f64::INFINITY,
2053                count: 1.0,
2054                boundary_rule: BoundaryRule::OpenLeft,
2055            }),
2056            f64::INFINITY
2057        );
2058        assert!(
2059            histogram
2060                .bucket_midpoint(&Bucket {
2061                    lower: f64::NEG_INFINITY,
2062                    upper: f64::INFINITY,
2063                    count: 1.0,
2064                    boundary_rule: BoundaryRule::OpenLeft,
2065                })
2066                .is_nan()
2067        );
2068    }
2069
2070    #[test]
2071    fn fraction_excludes_nan_observations() {
2072        let mut histogram = histogram(
2073            vec![Span {
2074                offset: 0,
2075                length: 1,
2076            }],
2077            vec![8.0],
2078        );
2079        histogram.count = 10.0;
2080        histogram.sum = f64::NAN;
2081
2082        assert_eq!(histogram.fraction(f64::NEG_INFINITY, f64::INFINITY), 0.8);
2083    }
2084
2085    #[test]
2086    fn subtraction_returns_gauge_histogram() {
2087        let left = histogram(
2088            vec![Span {
2089                offset: 0,
2090                length: 1,
2091            }],
2092            vec![3.0],
2093        );
2094        let right = histogram(
2095            vec![Span {
2096                offset: 0,
2097                length: 1,
2098            }],
2099            vec![1.0],
2100        );
2101
2102        let result = left.sub(&right).unwrap();
2103        assert_eq!(result.reset_hint, CounterResetHint::Gauge);
2104        assert_eq!(result.positive_buckets, vec![2.0]);
2105    }
2106
2107    #[test]
2108    fn subtraction_treats_compatible_empty_left_as_zero() {
2109        let left = histogram(vec![], vec![]);
2110        let right = histogram(
2111            vec![Span {
2112                offset: 0,
2113                length: 1,
2114            }],
2115            vec![2.0],
2116        );
2117
2118        let result = left.sub(&right).unwrap();
2119        assert_eq!(result.reset_hint, CounterResetHint::Gauge);
2120        assert_eq!(result.count, -2.0);
2121        assert_eq!(result.sum, -2.0);
2122        assert_eq!(result.positive_buckets, vec![-2.0]);
2123    }
2124
2125    #[test]
2126    fn subtraction_rejects_incompatible_empty_left() {
2127        let left = histogram(vec![], vec![]);
2128        let mut right = histogram(
2129            vec![Span {
2130                offset: 0,
2131                length: 1,
2132            }],
2133            vec![2.0],
2134        );
2135        right.schema = CUSTOM_BUCKETS_SCHEMA;
2136        right.custom_values = vec![1.0];
2137
2138        assert!(left.sub(&right).is_none());
2139    }
2140}
2141
2142// ---------------------------------------------------------------------------
2143// Stable Parquet field ids for native-histogram sub-fields.
2144//
2145// External readers resolve nested struct fields by `PARQUET:field_id`, so each
2146// sub-field (and list element) needs a stable positive id. The struct schema
2147// is fixed (always the same 18 fields). Each histogram column owns a block of
2148// ids (offset from a reserved base by the column's id), so several histogram
2149// columns in one table get disjoint sub-field ids. The reserved base is
2150// disjoint from user column ids and mito2 internal ids (`1 << 30`).
2151//
2152// The id is computed with checked arithmetic: the reserved base plus
2153// `column_id * stride` cannot always fit in a positive `i32` (a `ColumnId` is
2154// `u32`), so derivation returns `None` once the representable range is
2155// exceeded. Callers must handle `None` explicitly — the SST parquet writer
2156// surfaces it as an error rather than wrapping, panicking, or silently
2157// dropping the field id.
2158// ---------------------------------------------------------------------------
2159
2160/// Reserved base for native-histogram struct sub-field ids.
2161pub const NATIVE_HISTOGRAM_SUBFIELD_ID_BASE: i32 = 0x5000_0000;
2162
2163/// Number of ids reserved per histogram column (18 sub-fields + headroom for
2164/// list element ids), so multiple histogram columns get disjoint ids.
2165pub const NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE: i32 = 64;
2166
2167/// Offset of list element ids within a column's id block.
2168pub const NATIVE_HISTOGRAM_LIST_ELEMENT_OFFSET: i32 = 32;
2169
2170/// Returns the stable field id for a native-histogram struct sub-field,
2171/// namespaced by its parent `column_id`, or `None` if `name` is not a known
2172/// sub-field or the derived id overflows a positive `i32`.
2173pub fn native_histogram_subfield_id(column_id: i32, name: &str) -> Option<i32> {
2174    let idx = subfield_index(name)?;
2175    NATIVE_HISTOGRAM_SUBFIELD_ID_BASE
2176        .checked_add(column_id.checked_mul(NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE)?)
2177        .and_then(|v| v.checked_add(idx))
2178}
2179
2180/// Returns the stable list `element-id` for a list-typed native-histogram
2181/// sub-field, namespaced by its parent `column_id`, or `None` if `name` is not
2182/// a known sub-field or the derived id overflows a positive `i32`.
2183pub fn native_histogram_list_element_id(column_id: i32, name: &str) -> Option<i32> {
2184    let idx = subfield_index(name)?;
2185    NATIVE_HISTOGRAM_SUBFIELD_ID_BASE
2186        .checked_add(column_id.checked_mul(NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE)?)
2187        .and_then(|v| v.checked_add(NATIVE_HISTOGRAM_LIST_ELEMENT_OFFSET))
2188        .and_then(|v| v.checked_add(idx))
2189}
2190
2191fn subfield_index(name: &str) -> Option<i32> {
2192    NATIVE_HISTOGRAM_FIELD_NAMES
2193        .iter()
2194        .position(|n| *n == name)
2195        .map(|i| i as i32)
2196}
2197
2198#[cfg(test)]
2199mod subfield_id_tests {
2200    use super::*;
2201
2202    #[test]
2203    fn subfield_ids_are_namespaced_and_disjoint() {
2204        // SCHEMA=0, ZERO_THRESHOLD=1, SUM=2, ..., CUSTOM_VALUES=5.
2205        let sum_idx = 2;
2206        let custom_values_idx = 5;
2207
2208        // Ids are offset from the reserved base by the parent column id and
2209        // the sub-field index.
2210        assert_eq!(
2211            native_histogram_subfield_id(1, SUM_FIELD),
2212            Some(NATIVE_HISTOGRAM_SUBFIELD_ID_BASE + NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE + sum_idx,)
2213        );
2214        // List element ids additionally carry the list offset.
2215        assert_eq!(
2216            native_histogram_list_element_id(1, CUSTOM_VALUES_FIELD),
2217            Some(
2218                NATIVE_HISTOGRAM_SUBFIELD_ID_BASE
2219                    + NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE
2220                    + NATIVE_HISTOGRAM_LIST_ELEMENT_OFFSET
2221                    + custom_values_idx,
2222            )
2223        );
2224        // Different parent columns get disjoint ids for the same sub-field.
2225        assert_ne!(
2226            native_histogram_subfield_id(1, SUM_FIELD),
2227            native_histogram_subfield_id(7, SUM_FIELD)
2228        );
2229        // Unknown sub-field name -> None.
2230        assert_eq!(native_histogram_subfield_id(1, "not_a_field"), None);
2231    }
2232
2233    #[test]
2234    fn subfield_ids_overflow_returns_none() {
2235        // `column_id` is u32-sized, but the derived id must fit in a positive
2236        // i32. At column_id = 12_582_912, BASE + column_id*64 == i32::MAX + 1,
2237        // which previously overflowed (debug panic / release wrap). Checked
2238        // arithmetic must yield None instead of wrapping or panicking.
2239        assert_eq!(native_histogram_subfield_id(12_582_912, SUM_FIELD), None);
2240        assert_eq!(
2241            native_histogram_list_element_id(12_582_912, CUSTOM_VALUES_FIELD),
2242            None
2243        );
2244        // One below that boundary is still representable.
2245        assert!(native_histogram_subfield_id(12_582_911, SUM_FIELD).is_some());
2246    }
2247}