1use std::collections::BTreeMap;
23use std::sync::Arc;
24
25use datafusion::arrow::array::{
26 Array, ArrayRef, Float64Array, Int32Array, ListArray, PrimitiveArray, StructArray,
27 TimestampMillisecondArray, UInt64Array,
28};
29use datafusion::arrow::buffer::NullBuffer;
30use datafusion::arrow::datatypes::{
31 ArrowPrimitiveType, DataType as ArrowDataType, Field, Float64Type, Int32Type, Int64Type,
32 TimestampMillisecondType, UInt32Type, UInt64Type,
33};
34use datafusion_common::{DataFusionError, Result as DfResult};
35use datatypes::data_type::{ConcreteDataType, DataType};
36use datatypes::types::{StructField, StructType};
37use once_cell::sync::Lazy;
38
39use crate::prelude::greptime_native_histogram;
40use crate::prometheus::format_prometheus_float;
41
42pub const NATIVE_HISTOGRAM_FIELD: &str = "greptime_native_histogram";
43pub const SCHEMA_FIELD: &str = "schema";
44pub const ZERO_THRESHOLD_FIELD: &str = "zero_threshold";
45pub const SUM_FIELD: &str = "sum";
46pub const RESET_HINT_FIELD: &str = "reset_hint";
47pub const START_TIMESTAMP_FIELD: &str = "start_timestamp";
48pub const CUSTOM_VALUES_FIELD: &str = "custom_values";
49pub const POSITIVE_SPAN_OFFSETS_FIELD: &str = "positive_span_offsets";
50pub const POSITIVE_SPAN_LENGTHS_FIELD: &str = "positive_span_lengths";
51pub const NEGATIVE_SPAN_OFFSETS_FIELD: &str = "negative_span_offsets";
52pub const NEGATIVE_SPAN_LENGTHS_FIELD: &str = "negative_span_lengths";
53pub const COUNT_U64_FIELD: &str = "count_u64";
54pub const ZERO_COUNT_U64_FIELD: &str = "zero_count_u64";
55pub const POSITIVE_BUCKETS_I64_FIELD: &str = "positive_buckets_i64";
56pub const NEGATIVE_BUCKETS_I64_FIELD: &str = "negative_buckets_i64";
57pub const COUNT_F64_FIELD: &str = "count_f64";
58pub const ZERO_COUNT_F64_FIELD: &str = "zero_count_f64";
59pub const POSITIVE_BUCKETS_F64_FIELD: &str = "positive_buckets_f64";
60pub const NEGATIVE_BUCKETS_F64_FIELD: &str = "negative_buckets_f64";
61
62pub const NATIVE_HISTOGRAM_FIELD_NAMES: &[&str] = &[
65 SCHEMA_FIELD,
66 ZERO_THRESHOLD_FIELD,
67 SUM_FIELD,
68 RESET_HINT_FIELD,
69 START_TIMESTAMP_FIELD,
70 CUSTOM_VALUES_FIELD,
71 POSITIVE_SPAN_OFFSETS_FIELD,
72 POSITIVE_SPAN_LENGTHS_FIELD,
73 NEGATIVE_SPAN_OFFSETS_FIELD,
74 NEGATIVE_SPAN_LENGTHS_FIELD,
75 COUNT_U64_FIELD,
76 ZERO_COUNT_U64_FIELD,
77 POSITIVE_BUCKETS_I64_FIELD,
78 NEGATIVE_BUCKETS_I64_FIELD,
79 COUNT_F64_FIELD,
80 ZERO_COUNT_F64_FIELD,
81 POSITIVE_BUCKETS_F64_FIELD,
82 NEGATIVE_BUCKETS_F64_FIELD,
83];
84
85static NATIVE_HISTOGRAM_VALUE_TYPE: Lazy<ConcreteDataType> = Lazy::new(|| {
86 let fields = NATIVE_HISTOGRAM_FIELD_NAMES
87 .iter()
88 .filter_map(|name| {
89 let data_type = native_histogram_field_type(name)?;
90 Some(StructField::new((*name).to_string(), data_type, true))
91 })
92 .collect();
93 ConcreteDataType::struct_datatype(StructType::new(Arc::new(fields)))
94});
95
96pub fn native_histogram_field_type(name: &str) -> Option<ConcreteDataType> {
98 match name {
99 SCHEMA_FIELD | RESET_HINT_FIELD => Some(ConcreteDataType::int32_datatype()),
100 ZERO_THRESHOLD_FIELD | SUM_FIELD | COUNT_F64_FIELD | ZERO_COUNT_F64_FIELD => {
101 Some(ConcreteDataType::float64_datatype())
102 }
103 START_TIMESTAMP_FIELD => Some(ConcreteDataType::timestamp_millisecond_datatype()),
104 CUSTOM_VALUES_FIELD | POSITIVE_BUCKETS_F64_FIELD | NEGATIVE_BUCKETS_F64_FIELD => Some(
105 ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::float64_datatype())),
106 ),
107 POSITIVE_SPAN_OFFSETS_FIELD | NEGATIVE_SPAN_OFFSETS_FIELD => Some(
108 ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype())),
109 ),
110 POSITIVE_SPAN_LENGTHS_FIELD | NEGATIVE_SPAN_LENGTHS_FIELD => Some(
111 ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::uint32_datatype())),
112 ),
113 COUNT_U64_FIELD | ZERO_COUNT_U64_FIELD => Some(ConcreteDataType::uint64_datatype()),
114 POSITIVE_BUCKETS_I64_FIELD | NEGATIVE_BUCKETS_I64_FIELD => Some(
115 ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int64_datatype())),
116 ),
117 _ => None,
118 }
119}
120
121pub fn native_histogram_value_type() -> &'static ConcreteDataType {
123 &NATIVE_HISTOGRAM_VALUE_TYPE
124}
125
126pub fn is_native_histogram_value_type(data_type: &ConcreteDataType) -> bool {
128 data_type == native_histogram_value_type()
129}
130
131pub fn is_native_histogram_value_schema(name: &str, data_type: &ConcreteDataType) -> bool {
133 name == greptime_native_histogram() && is_native_histogram_value_type(data_type)
134}
135
136pub const CUSTOM_BUCKETS_SCHEMA: i32 = -53;
138const MIN_EXPONENTIAL_SCHEMA: i32 = -4;
139const MAX_EXPONENTIAL_SCHEMA: i32 = 8;
140
141#[derive(Clone, Copy, Debug, PartialEq, Eq)]
146pub enum CounterResetHint {
147 Unknown,
149 CounterReset,
151 NotCounterReset,
153 Gauge,
155 Unrecognized(i32),
157}
158
159impl From<i32> for CounterResetHint {
160 fn from(value: i32) -> Self {
161 match value {
162 0 => Self::Unknown,
163 1 => Self::CounterReset,
164 2 => Self::NotCounterReset,
165 3 => Self::Gauge,
166 value => Self::Unrecognized(value),
167 }
168 }
169}
170
171impl From<CounterResetHint> for i32 {
172 fn from(value: CounterResetHint) -> Self {
173 match value {
174 CounterResetHint::Unknown => 0,
175 CounterResetHint::CounterReset => 1,
176 CounterResetHint::NotCounterReset => 2,
177 CounterResetHint::Gauge => 3,
178 CounterResetHint::Unrecognized(value) => value,
179 }
180 }
181}
182
183pub const UNKNOWN_COUNTER_RESET_HINT: CounterResetHint = CounterResetHint::Unknown;
185pub const COUNTER_RESET_HINT: CounterResetHint = CounterResetHint::CounterReset;
187pub const NOT_COUNTER_RESET_HINT: CounterResetHint = CounterResetHint::NotCounterReset;
189pub const GAUGE_RESET_HINT: CounterResetHint = CounterResetHint::Gauge;
191
192#[derive(Clone, Debug, PartialEq)]
194pub struct Span {
195 pub offset: i32,
197 pub length: u32,
199}
200
201#[derive(Clone, Copy, Debug, PartialEq, Eq)]
203enum BoundaryRule {
204 OpenLeft = 0,
205 OpenRight = 1,
206 ClosedBoth = 3,
207}
208
209#[derive(Clone, Debug, PartialEq)]
211struct Bucket {
212 lower: f64,
213 upper: f64,
214 count: f64,
215 boundary_rule: BoundaryRule,
216}
217
218#[derive(Clone, Debug, PartialEq)]
223pub struct NativeHistogram {
224 pub schema: i32,
226 pub zero_threshold: f64,
228 pub sum: f64,
230 pub reset_hint: CounterResetHint,
232 pub start_timestamp: Option<i64>,
234 pub custom_values: Vec<f64>,
236 pub positive_spans: Vec<Span>,
238 pub negative_spans: Vec<Span>,
240 pub count: f64,
242 pub zero_count: f64,
244 pub positive_buckets: Vec<f64>,
246 pub negative_buckets: Vec<f64>,
248}
249
250impl NativeHistogram {
251 fn uses_custom_buckets(&self) -> bool {
252 self.schema == CUSTOM_BUCKETS_SCHEMA
253 }
254
255 fn compatible_with(&self, other: &Self) -> bool {
256 self.schema == other.schema
257 && self.zero_threshold == other.zero_threshold
258 && self.custom_values == other.custom_values
259 }
260
261 pub fn zero_like(&self) -> Self {
263 let mut result = self.clone();
264 result.count = 0.0;
265 result.zero_count = 0.0;
266 result.sum = 0.0;
267 result.positive_buckets.fill(0.0);
268 result.negative_buckets.fill(0.0);
269 result
270 }
271
272 fn combine_exact(
273 &self,
274 other: &Self,
275 reset_hint: CounterResetHint,
276 op: impl Fn(f64, f64) -> f64 + Copy,
277 ) -> Option<Self> {
278 if !self.compatible_with(other) {
279 return None;
280 }
281
282 let mut result = self.clone();
283 result.count = op(result.count, other.count);
284 result.zero_count = op(result.zero_count, other.zero_count);
285 result.sum = op(result.sum, other.sum);
286 result.reset_hint = reset_hint;
287 (result.positive_spans, result.positive_buckets) = merge_side(
288 &self.positive_spans,
289 &self.positive_buckets,
290 &other.positive_spans,
291 &other.positive_buckets,
292 op,
293 )?;
294 (result.negative_spans, result.negative_buckets) = merge_side(
295 &self.negative_spans,
296 &self.negative_buckets,
297 &other.negative_spans,
298 &other.negative_buckets,
299 op,
300 )?;
301 Some(result)
302 }
303
304 pub fn add(&self, other: &Self) -> Option<Self> {
308 let reset_hint = add_reset_hint(self.reset_hint, other.reset_hint);
309 let (left, right) = self.reconcile(other)?;
310 left.combine_exact(&right, reset_hint, |left, right| left + right)?
311 .compact()
312 }
313
314 pub fn sub(&self, other: &Self) -> Option<Self> {
318 let (left, right) = self.reconcile(other)?;
319 left.combine_exact(&right, CounterResetHint::Gauge, |left, right| left - right)?
320 .compact()
321 }
322
323 pub fn negated(self) -> Self {
325 self.scale(-1.0)
326 }
327
328 pub fn into_gauge(mut self) -> Self {
330 self.reset_hint = CounterResetHint::Gauge;
331 self
332 }
333
334 pub fn counter_reset_hints_contradict(&self, other: &Self) -> bool {
336 matches!(
337 (self.reset_hint, other.reset_hint),
338 (
339 CounterResetHint::CounterReset,
340 CounterResetHint::NotCounterReset
341 ) | (
342 CounterResetHint::NotCounterReset,
343 CounterResetHint::CounterReset
344 )
345 )
346 }
347
348 pub fn needs_custom_reconciliation(&self, other: &Self) -> bool {
350 self.uses_custom_buckets()
351 && other.uses_custom_buckets()
352 && self.custom_values != other.custom_values
353 }
354
355 pub fn promql_eq(&self, other: &Self) -> bool {
359 self.schema == other.schema
360 && self.zero_threshold == other.zero_threshold
361 && self.custom_values == other.custom_values
362 && self.count.to_bits() == other.count.to_bits()
363 && self.zero_count.to_bits() == other.zero_count.to_bits()
364 && self.sum.to_bits() == other.sum.to_bits()
365 && side_layout_equal(
366 &self.positive_spans,
367 &self.positive_buckets,
368 &other.positive_spans,
369 &other.positive_buckets,
370 )
371 && side_layout_equal(
372 &self.negative_spans,
373 &self.negative_buckets,
374 &other.negative_spans,
375 &other.negative_buckets,
376 )
377 }
378
379 pub fn promql_string(&self) -> String {
381 let mut parts = vec![format!("count:{}", self.count), format!("sum:{}", self.sum)];
382 if let Some(buckets) = self.all_buckets() {
383 parts.extend(
384 buckets
385 .into_iter()
386 .filter(|bucket| bucket.count != 0.0)
387 .map(|bucket| {
388 let (left, right) = match bucket.boundary_rule {
389 BoundaryRule::OpenLeft => ("(", "]"),
390 BoundaryRule::OpenRight => ("[", ")"),
391 BoundaryRule::ClosedBoth => ("[", "]"),
392 };
393 format!(
394 "{}{},{}{}:{}",
395 left, bucket.lower, bucket.upper, right, bucket.count
396 )
397 }),
398 );
399 }
400 format!("{{{}}}", parts.join(", "))
401 }
402
403 pub fn estimated_stdvar(&self) -> f64 {
405 if self.count == 0.0 {
406 return f64::NAN;
407 }
408 let mean = self.sum / self.count;
409 let Some(buckets) = self.all_buckets() else {
410 return f64::NAN;
411 };
412 buckets
413 .into_iter()
414 .map(|bucket| {
415 let midpoint = self.bucket_midpoint(&bucket);
416 bucket.count * (midpoint - mean).powi(2)
417 })
418 .sum::<f64>()
419 / self.count
420 }
421
422 pub fn estimated_stddev(&self) -> f64 {
424 self.estimated_stdvar().sqrt()
425 }
426
427 pub fn scale(mut self, factor: f64) -> Self {
431 self.count *= factor;
432 self.zero_count *= factor;
433 self.sum *= factor;
434 for count in &mut self.positive_buckets {
435 *count *= factor;
436 }
437 for count in &mut self.negative_buckets {
438 *count *= factor;
439 }
440 if factor < 0.0 {
441 self.reset_hint = CounterResetHint::Gauge;
442 }
443 self
444 }
445
446 pub fn divide_by(mut self, divisor: f64) -> Self {
450 self.count /= divisor;
451 self.zero_count /= divisor;
452 self.sum /= divisor;
453 if divisor == 0.0 {
454 self.positive_spans.clear();
455 self.positive_buckets.clear();
456 self.negative_spans.clear();
457 self.negative_buckets.clear();
458 } else {
459 for count in &mut self.positive_buckets {
460 *count /= divisor;
461 }
462 for count in &mut self.negative_buckets {
463 *count /= divisor;
464 }
465 }
466 if divisor < 0.0 {
467 self.reset_hint = CounterResetHint::Gauge;
468 }
469 self
470 }
471
472 fn compact(mut self) -> Option<Self> {
473 let (spans, buckets) = compact_side(&self.positive_spans, &self.positive_buckets)?;
474 self.positive_spans = spans;
475 self.positive_buckets = buckets;
476 let (spans, buckets) = compact_side(&self.negative_spans, &self.negative_buckets)?;
477 self.negative_spans = spans;
478 self.negative_buckets = buckets;
479 Some(self)
480 }
481
482 pub fn detect_reset(&self, previous: &Self) -> bool {
484 match self.reset_hint {
485 CounterResetHint::CounterReset => return true,
486 CounterResetHint::NotCounterReset => return false,
487 CounterResetHint::Unknown
488 | CounterResetHint::Gauge
489 | CounterResetHint::Unrecognized(_) => {}
490 }
491
492 if self.count < previous.count {
493 return true;
494 }
495
496 match (self.uses_custom_buckets(), previous.uses_custom_buckets()) {
497 (true, true) => {
498 let Some((current, previous)) = reconcile_custom(self, previous) else {
499 return true;
500 };
501 current.zero_count < previous.zero_count
502 || current.side_has_reset(true, &previous)
503 || current.side_has_reset(false, &previous)
504 }
505 (true, false) | (false, true) => true,
506 (false, false) => {
507 if self.schema > previous.schema || self.zero_threshold < previous.zero_threshold {
508 return true;
509 }
510
511 let mut previous = previous.clone();
512 if self.zero_threshold > previous.zero_threshold {
513 let Some(expanded) = previous.expanded_zero_threshold(self.zero_threshold)
514 else {
515 return true;
516 };
517 if expanded != self.zero_threshold
518 || previous.grow_zero_threshold(self.zero_threshold).is_none()
519 {
520 return true;
521 }
522 }
523 let Some(previous) = previous.copy_to_schema(self.schema) else {
524 return true;
525 };
526
527 self.zero_count < previous.zero_count
528 || self.side_has_reset(true, &previous)
529 || self.side_has_reset(false, &previous)
530 }
531 }
532 }
533
534 fn detect_start_timestamp_reset(
535 &self,
536 previous: &Self,
537 previous_ts: i64,
538 current_ts: i64,
539 ) -> bool {
540 let current_start = self.start_timestamp.unwrap_or_default();
541 if current_start == 0 || current_start >= current_ts || current_start < previous_ts {
542 return false;
543 }
544 if current_start > previous_ts {
545 return true;
546 }
547
548 let previous_start = previous.start_timestamp.unwrap_or_default();
549 previous_start <= previous_ts && previous_start != 0 && previous_start != previous_ts
550 }
551
552 pub fn detect_counter_reset(&self, previous: &Self, previous_ts: i64, current_ts: i64) -> bool {
554 self.detect_start_timestamp_reset(previous, previous_ts, current_ts)
555 || self.detect_reset(previous)
556 }
557
558 fn all_buckets(&self) -> Option<Vec<Bucket>> {
559 let mut buckets = self.side_buckets(false)?;
560 buckets.reverse();
561 if self.zero_count != 0.0 {
562 buckets.push(Bucket {
563 lower: -self.zero_threshold,
564 upper: self.zero_threshold,
565 count: self.zero_count,
566 boundary_rule: BoundaryRule::ClosedBoth,
567 });
568 }
569 buckets.extend(self.side_buckets(true)?);
570 Some(buckets)
571 }
572
573 fn side_buckets(&self, positive: bool) -> Option<Vec<Bucket>> {
574 let (spans, counts) = if positive {
575 (&self.positive_spans, &self.positive_buckets)
576 } else {
577 (&self.negative_spans, &self.negative_buckets)
578 };
579
580 let mut result = Vec::with_capacity(counts.len());
581 for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(counts) {
582 let upper = get_bound(idx, self.schema, &self.custom_values)?;
583 let lower = get_bound(idx.checked_sub(1)?, self.schema, &self.custom_values)?;
584 if positive {
585 result.push(Bucket {
586 lower,
587 upper,
588 count: *count,
589 boundary_rule: if self.uses_custom_buckets() && idx == 0 {
590 BoundaryRule::ClosedBoth
591 } else {
592 BoundaryRule::OpenLeft
593 },
594 });
595 } else {
596 result.push(Bucket {
597 lower: -upper,
598 upper: -lower,
599 count: *count,
600 boundary_rule: BoundaryRule::OpenRight,
601 });
602 }
603 }
604 Some(result)
605 }
606
607 pub fn quantile(&self, q: f64) -> f64 {
612 if q < 0.0 {
613 return f64::NEG_INFINITY;
614 }
615 if q > 1.0 {
616 return f64::INFINITY;
617 }
618 if self.count == 0.0 || q.is_nan() {
619 return f64::NAN;
620 }
621
622 let Some(mut buckets) = self.all_buckets() else {
623 return f64::NAN;
624 };
625 let rank = q * self.count;
626 let mut count = 0.0;
627 for bucket in &mut buckets {
628 if bucket.count == 0.0 {
629 continue;
630 }
631 count += bucket.count;
632 if count < rank {
633 continue;
634 }
635
636 if !self.uses_custom_buckets() && bucket.lower < 0.0 && bucket.upper > 0.0 {
637 if self.negative_buckets.is_empty() && !self.positive_buckets.is_empty() {
638 bucket.lower = 0.0;
639 } else if self.positive_buckets.is_empty() && !self.negative_buckets.is_empty() {
640 bucket.upper = 0.0;
641 }
642 } else if self.uses_custom_buckets() {
643 if bucket.lower == f64::NEG_INFINITY {
644 if bucket.upper <= 0.0 {
645 return bucket.upper;
646 }
647 bucket.lower = 0.0;
648 } else if bucket.upper == f64::INFINITY {
649 return bucket.lower;
650 }
651 }
652
653 let rank_in_bucket = rank - (count - bucket.count);
654 let fraction = rank_in_bucket / bucket.count;
655 if self.uses_custom_buckets() || (bucket.lower <= 0.0 && bucket.upper >= 0.0) {
656 return bucket.lower + (bucket.upper - bucket.lower) * fraction;
657 }
658
659 let log_lower = bucket.lower.abs().log2();
660 let log_upper = bucket.upper.abs().log2();
661 if bucket.lower > 0.0 {
662 return 2.0_f64.powf(log_lower + (log_upper - log_lower) * fraction);
663 }
664 return -2.0_f64.powf(log_upper + (log_lower - log_upper) * (1.0 - fraction));
665 }
666
667 f64::NAN
668 }
669
670 pub fn fraction(&self, lower: f64, upper: f64) -> f64 {
672 if self.count == 0.0 || lower.is_nan() || upper.is_nan() {
673 return f64::NAN;
674 }
675 if lower >= upper {
676 return 0.0;
677 }
678
679 let Some(mut buckets) = self.all_buckets() else {
680 return f64::NAN;
681 };
682 let count = if self.sum.is_nan() {
683 buckets.iter().map(|bucket| bucket.count).sum()
684 } else {
685 self.count
686 };
687
688 let mut rank = 0.0;
689 let mut lower_rank = 0.0;
690 let mut upper_rank = 0.0;
691 let mut lower_set = false;
692 let mut upper_set = false;
693
694 for bucket in &mut buckets {
695 let zero_bucket = bucket.lower <= 0.0 && bucket.upper >= 0.0;
696 if zero_bucket {
697 if self.negative_buckets.is_empty() && !self.positive_buckets.is_empty() {
698 bucket.lower = 0.0;
699 } else if self.positive_buckets.is_empty() && !self.negative_buckets.is_empty() {
700 bucket.upper = 0.0;
701 }
702 }
703
704 if !lower_set && bucket.lower >= lower {
705 lower_rank = rank;
706 lower_set = true;
707 }
708 if !upper_set && bucket.lower >= upper {
709 upper_rank = rank;
710 upper_set = true;
711 }
712 if lower_set && upper_set {
713 break;
714 }
715 if !lower_set && bucket.lower < lower && bucket.upper > lower {
716 lower_rank = self.interpolate_rank(bucket, rank, lower, zero_bucket);
717 lower_set = true;
718 }
719 if !upper_set && bucket.lower < upper && bucket.upper > upper {
720 upper_rank = self.interpolate_rank(bucket, rank, upper, zero_bucket);
721 upper_set = true;
722 }
723 if lower_set && upper_set {
724 break;
725 }
726 rank += bucket.count;
727 }
728
729 if !lower_set || lower_rank > count {
730 lower_rank = count;
731 }
732 if !upper_set || upper_rank > count {
733 upper_rank = count;
734 }
735
736 (upper_rank - lower_rank) / self.count
737 }
738
739 pub fn to_prometheus_buckets(&self) -> Option<Vec<(u8, String, String, String)>> {
741 Some(
742 self.all_buckets()?
743 .into_iter()
744 .filter(|bucket| bucket.count != 0.0)
745 .map(|bucket| {
746 (
747 bucket.boundary_rule as u8,
748 format_prometheus_float(bucket.lower),
749 format_prometheus_float(bucket.upper),
750 format_prometheus_float(bucket.count),
751 )
752 })
753 .collect(),
754 )
755 }
756
757 fn interpolate_rank(&self, bucket: &Bucket, rank: f64, value: f64, zero_bucket: bool) -> f64 {
758 if self.uses_custom_buckets() || zero_bucket {
759 if bucket.lower == f64::NEG_INFINITY {
760 return bucket.count;
761 }
762 return rank + bucket.count * (value - bucket.lower) / (bucket.upper - bucket.lower);
763 }
764
765 let log_lower = bucket.lower.abs().log2();
766 let log_upper = bucket.upper.abs().log2();
767 let log_value = value.abs().log2();
768 let fraction = if value > 0.0 {
769 (log_value - log_lower) / (log_upper - log_lower)
770 } else {
771 1.0 - ((log_value - log_upper) / (log_lower - log_upper))
772 };
773 rank + bucket.count * fraction
774 }
775
776 fn bucket_midpoint(&self, bucket: &Bucket) -> f64 {
777 if self.uses_custom_buckets() {
778 return (bucket.lower + bucket.upper) / 2.0;
779 }
780 if bucket.lower <= 0.0 && bucket.upper >= 0.0 {
781 return 0.0;
782 }
783 if bucket.upper < 0.0 {
784 -((bucket.lower.abs() * bucket.upper.abs()).sqrt())
785 } else {
786 (bucket.lower * bucket.upper).sqrt()
787 }
788 }
789
790 fn side_has_reset(&self, positive: bool, previous: &Self) -> bool {
791 let (current_spans, current_buckets, previous_spans, previous_buckets) = if positive {
792 (
793 &self.positive_spans,
794 &self.positive_buckets,
795 &previous.positive_spans,
796 &previous.positive_buckets,
797 )
798 } else {
799 (
800 &self.negative_spans,
801 &self.negative_buckets,
802 &previous.negative_spans,
803 &previous.negative_buckets,
804 )
805 };
806 let Some(current) = side_counts(current_spans, current_buckets) else {
807 return true;
808 };
809 let Some(previous) = side_counts(previous_spans, previous_buckets) else {
810 return true;
811 };
812 previous.keys().chain(current.keys()).any(|idx| {
813 current.get(idx).copied().unwrap_or_default()
814 < previous.get(idx).copied().unwrap_or_default()
815 })
816 }
817
818 fn reconcile(&self, other: &Self) -> Option<(Self, Self)> {
819 match (self.uses_custom_buckets(), other.uses_custom_buckets()) {
820 (true, true) => reconcile_custom(self, other),
821 (false, false) => reconcile_exponential(self, other),
822 _ => None,
823 }
824 }
825}
826
827fn reconcile_exponential(
828 left: &NativeHistogram,
829 right: &NativeHistogram,
830) -> Option<(NativeHistogram, NativeHistogram)> {
831 let schema = left.schema.min(right.schema);
832 let mut left = left.copy_to_schema(schema)?;
833 let mut right = right.copy_to_schema(schema)?;
834 let mut zero_threshold = left.zero_threshold.max(right.zero_threshold);
835 loop {
836 let expanded = left
837 .expanded_zero_threshold(zero_threshold)?
838 .max(right.expanded_zero_threshold(zero_threshold)?);
839 if expanded == zero_threshold {
840 break;
841 }
842 zero_threshold = expanded;
843 }
844 left.grow_zero_threshold(zero_threshold)?;
845 right.grow_zero_threshold(zero_threshold)?;
846 Some((left.compact()?, right.compact()?))
847}
848
849fn reconcile_custom(
850 left: &NativeHistogram,
851 right: &NativeHistogram,
852) -> Option<(NativeHistogram, NativeHistogram)> {
853 let custom_values = if left.custom_values == right.custom_values {
854 left.custom_values.clone()
855 } else {
856 left.custom_values
857 .iter()
858 .copied()
859 .filter(|value| right.custom_values.contains(value))
860 .collect()
861 };
862
863 Some((
864 left.copy_to_custom_values(custom_values.clone())?
865 .compact()?,
866 right.copy_to_custom_values(custom_values)?.compact()?,
867 ))
868}
869
870impl NativeHistogram {
871 fn copy_to_schema(&self, target_schema: i32) -> Option<Self> {
872 if self.uses_custom_buckets()
873 || !(MIN_EXPONENTIAL_SCHEMA..=MAX_EXPONENTIAL_SCHEMA).contains(&target_schema)
874 || target_schema > self.schema
875 {
876 return None;
877 }
878 if target_schema == self.schema {
879 return Some(self.clone());
880 }
881
882 let mut result = self.clone();
883 result.schema = target_schema;
884 (result.positive_spans, result.positive_buckets) = reduce_side(
885 &self.positive_spans,
886 &self.positive_buckets,
887 self.schema,
888 target_schema,
889 )?;
890 (result.negative_spans, result.negative_buckets) = reduce_side(
891 &self.negative_spans,
892 &self.negative_buckets,
893 self.schema,
894 target_schema,
895 )?;
896 Some(result)
897 }
898
899 fn copy_to_custom_values(&self, custom_values: Vec<f64>) -> Option<Self> {
900 if !self.uses_custom_buckets() {
901 return None;
902 }
903 if self.custom_values == custom_values {
904 return Some(self.clone());
905 }
906
907 let mut result = self.clone();
908 result.custom_values = custom_values.clone();
909 result.negative_spans.clear();
910 result.negative_buckets.clear();
911 (result.positive_spans, result.positive_buckets) = map_custom_side(
912 &self.positive_spans,
913 &self.positive_buckets,
914 &self.custom_values,
915 &custom_values,
916 )?;
917 Some(result)
918 }
919
920 fn grow_zero_threshold(&mut self, zero_threshold: f64) -> Option<()> {
921 if self.uses_custom_buckets() || zero_threshold == self.zero_threshold {
922 self.zero_threshold = zero_threshold;
923 return Some(());
924 }
925
926 let (spans, buckets, zero_count) = fold_zero_side(
927 &self.positive_spans,
928 &self.positive_buckets,
929 self.schema,
930 zero_threshold,
931 )?;
932 self.positive_spans = spans;
933 self.positive_buckets = buckets;
934 self.zero_count += zero_count;
935
936 let (spans, buckets, zero_count) = fold_zero_side(
937 &self.negative_spans,
938 &self.negative_buckets,
939 self.schema,
940 zero_threshold,
941 )?;
942 self.negative_spans = spans;
943 self.negative_buckets = buckets;
944 self.zero_count += zero_count;
945 self.zero_threshold = zero_threshold;
946 Some(())
947 }
948
949 fn expanded_zero_threshold(&self, mut zero_threshold: f64) -> Option<f64> {
950 if self.uses_custom_buckets() {
951 return Some(zero_threshold);
952 }
953 zero_threshold = expand_zero_threshold_side(
954 &self.positive_spans,
955 &self.positive_buckets,
956 self.schema,
957 zero_threshold,
958 )?;
959 expand_zero_threshold_side(
960 &self.negative_spans,
961 &self.negative_buckets,
962 self.schema,
963 zero_threshold,
964 )
965 }
966}
967
968fn add_reset_hint(left: CounterResetHint, right: CounterResetHint) -> CounterResetHint {
969 if left == CounterResetHint::Gauge || right == CounterResetHint::Gauge {
970 CounterResetHint::Gauge
971 } else if left == right {
972 left
973 } else {
974 CounterResetHint::Unknown
975 }
976}
977
978fn side_bucket_indices(spans: &[Span]) -> Option<Vec<i32>> {
979 let mut indices = Vec::new();
980 let mut current_index = 0i32;
981 let mut first = true;
982 for (span_index, span) in spans.iter().enumerate() {
983 if span_index > 0 && span.offset < 0 {
984 return None;
985 }
986 if first {
987 current_index = span.offset;
988 first = false;
989 } else {
990 current_index = current_index.checked_add(span.offset)?;
991 }
992 for _ in 0..span.length {
993 indices.push(current_index);
994 current_index = current_index.checked_add(1)?;
995 }
996 }
997 Some(indices)
998}
999
1000fn span_bucket_len(spans: &[Span]) -> Option<usize> {
1001 spans
1002 .iter()
1003 .try_fold(0usize, |sum, span| sum.checked_add(span.length as usize))
1004}
1005
1006fn spans_from_indices_counts(values: Vec<(i32, f64)>) -> Option<(Vec<Span>, Vec<f64>)> {
1007 let mut spans = Vec::<Span>::new();
1008 let mut buckets = Vec::new();
1009 let mut previous_index = None::<i32>;
1010
1011 for (idx, count) in values {
1012 if count == 0.0 {
1013 continue;
1014 }
1015 match (spans.last_mut(), previous_index) {
1016 (Some(span), Some(previous)) if previous.checked_add(1) == Some(idx) => {
1017 span.length = span.length.checked_add(1)?;
1018 }
1019 (_, Some(previous)) => {
1020 spans.push(Span {
1021 offset: idx.checked_sub(previous)?.checked_sub(1)?,
1022 length: 1,
1023 });
1024 }
1025 (_, None) => {
1026 spans.push(Span {
1027 offset: idx,
1028 length: 1,
1029 });
1030 }
1031 }
1032 buckets.push(count);
1033 previous_index = Some(idx);
1034 }
1035
1036 Some((spans, buckets))
1037}
1038
1039fn side_counts(spans: &[Span], buckets: &[f64]) -> Option<BTreeMap<i32, f64>> {
1040 let mut values = BTreeMap::new();
1041 for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1042 if *count != 0.0 {
1043 values.insert(idx, *count);
1044 }
1045 }
1046 Some(values)
1047}
1048
1049fn side_layout_equal(
1050 left_spans: &[Span],
1051 left_buckets: &[f64],
1052 right_spans: &[Span],
1053 right_buckets: &[f64],
1054) -> bool {
1055 if span_bucket_len(left_spans) != Some(left_buckets.len())
1056 || span_bucket_len(right_spans) != Some(right_buckets.len())
1057 {
1058 return false;
1059 }
1060 let Some(left_indices) = side_bucket_indices(left_spans) else {
1061 return false;
1062 };
1063 let Some(right_indices) = side_bucket_indices(right_spans) else {
1064 return false;
1065 };
1066 left_indices == right_indices
1067 && left_buckets
1068 .iter()
1069 .zip(right_buckets)
1070 .all(|(left, right)| left.to_bits() == right.to_bits())
1071}
1072
1073fn merge_side(
1074 left_spans: &[Span],
1075 left_buckets: &[f64],
1076 right_spans: &[Span],
1077 right_buckets: &[f64],
1078 op: impl Fn(f64, f64) -> f64,
1079) -> Option<(Vec<Span>, Vec<f64>)> {
1080 let left = side_counts(left_spans, left_buckets)?;
1081 let right = side_counts(right_spans, right_buckets)?;
1082 let mut values = BTreeMap::new();
1083 for idx in left.keys().chain(right.keys()) {
1084 values.insert(
1085 *idx,
1086 op(
1087 left.get(idx).copied().unwrap_or_default(),
1088 right.get(idx).copied().unwrap_or_default(),
1089 ),
1090 );
1091 }
1092 spans_from_indices_counts(values.into_iter().collect())
1093}
1094
1095fn reduce_side(
1096 spans: &[Span],
1097 buckets: &[f64],
1098 schema: i32,
1099 target_schema: i32,
1100) -> Option<(Vec<Span>, Vec<f64>)> {
1101 let factor = 1_i32.checked_shl((schema - target_schema) as u32)?;
1102 let mut values = std::collections::BTreeMap::<i32, f64>::new();
1103 for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1104 let target_idx = ceil_div(idx, factor);
1105 *values.entry(target_idx).or_default() += *count;
1106 }
1107 spans_from_indices_counts(values.into_iter().collect())
1108}
1109
1110fn compact_side(spans: &[Span], buckets: &[f64]) -> Option<(Vec<Span>, Vec<f64>)> {
1111 let indices = side_bucket_indices(spans)?;
1112 spans_from_indices_counts(indices.into_iter().zip(buckets.iter().copied()).collect())
1113}
1114
1115fn map_custom_side(
1116 spans: &[Span],
1117 buckets: &[f64],
1118 old_values: &[f64],
1119 new_values: &[f64],
1120) -> Option<(Vec<Span>, Vec<f64>)> {
1121 let mut values = std::collections::BTreeMap::<i32, f64>::new();
1122 for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1123 let upper = get_bound(idx, CUSTOM_BUCKETS_SCHEMA, old_values)?;
1124 let target_idx = new_values
1125 .iter()
1126 .position(|value| *value >= upper)
1127 .unwrap_or(new_values.len()) as i32;
1128 *values.entry(target_idx).or_default() += *count;
1129 }
1130 spans_from_indices_counts(values.into_iter().collect())
1131}
1132
1133fn fold_zero_side(
1134 spans: &[Span],
1135 buckets: &[f64],
1136 schema: i32,
1137 zero_threshold: f64,
1138) -> Option<(Vec<Span>, Vec<f64>, f64)> {
1139 let mut kept = Vec::new();
1140 let mut zero_count = 0.0;
1141 for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1142 if get_bound(idx, schema, &[])? <= zero_threshold {
1143 zero_count += *count;
1144 } else {
1145 kept.push((idx, *count));
1146 }
1147 }
1148 let (spans, buckets) = spans_from_indices_counts(kept)?;
1149 Some((spans, buckets, zero_count))
1150}
1151
1152fn expand_zero_threshold_side(
1153 spans: &[Span],
1154 buckets: &[f64],
1155 schema: i32,
1156 mut zero_threshold: f64,
1157) -> Option<f64> {
1158 for (idx, count) in side_bucket_indices(spans)?.into_iter().zip(buckets) {
1159 if *count == 0.0 {
1160 continue;
1161 }
1162 let lower = get_bound(idx.checked_sub(1)?, schema, &[])?;
1163 let upper = get_bound(idx, schema, &[])?;
1164 if lower < zero_threshold && zero_threshold < upper {
1165 zero_threshold = upper;
1166 }
1167 }
1168 Some(zero_threshold)
1169}
1170
1171fn ceil_div(value: i32, divisor: i32) -> i32 {
1172 value.div_euclid(divisor) + i32::from(value.rem_euclid(divisor) != 0)
1173}
1174
1175pub fn exponential_overflow_bucket_index(schema: i32) -> Option<i32> {
1177 if !(MIN_EXPONENTIAL_SCHEMA..=MAX_EXPONENTIAL_SCHEMA).contains(&schema) {
1178 return None;
1179 }
1180 let last_finite = if schema >= 0 {
1181 1024_i64.checked_shl(schema as u32)?
1182 } else {
1183 1024_i64.checked_shr((-schema) as u32)?
1184 };
1185 i32::try_from(last_finite.checked_add(1)?).ok()
1186}
1187
1188fn get_bound(idx: i32, schema: i32, custom_values: &[f64]) -> Option<f64> {
1189 if schema == CUSTOM_BUCKETS_SCHEMA {
1190 return match idx {
1191 -1 => Some(f64::NEG_INFINITY),
1192 idx if idx == custom_values.len() as i32 => Some(f64::INFINITY),
1193 idx if idx >= 0 && (idx as usize) < custom_values.len() => {
1194 Some(custom_values[idx as usize])
1195 }
1196 _ => None,
1197 };
1198 }
1199
1200 let overflow_index = exponential_overflow_bucket_index(schema)?;
1201 if idx > overflow_index {
1202 return None;
1203 }
1204 if idx == overflow_index {
1205 return Some(f64::INFINITY);
1206 }
1207 if idx == overflow_index - 1 {
1208 return Some(f64::MAX);
1209 }
1210 if schema < 0 {
1211 let exponent = i64::from(idx).checked_shl((-schema) as u32)?;
1212 let Ok(exponent) = i32::try_from(exponent) else {
1213 return Some(0.0);
1214 };
1215 return Some(2.0_f64.powi(exponent));
1216 }
1217 let exponent = idx as f64 / (1u32 << schema) as f64;
1218 Some(2.0_f64.powf(exponent))
1219}
1220
1221pub fn native_histogram_arrow_type() -> ArrowDataType {
1223 native_histogram_value_type().as_arrow_type()
1224}
1225
1226fn struct_child<'a>(array: &'a StructArray, name: &str) -> DfResult<&'a ArrayRef> {
1227 let index = array
1228 .fields()
1229 .iter()
1230 .position(|field| field.name() == name)
1231 .ok_or_else(|| {
1232 DataFusionError::Execution(format!("native histogram missing field {name}"))
1233 })?;
1234 Ok(array.column(index))
1235}
1236
1237fn primitive_child<'a, T>(array: &'a StructArray, name: &str) -> DfResult<&'a PrimitiveArray<T>>
1238where
1239 T: ArrowPrimitiveType,
1240{
1241 let child = struct_child(array, name)?;
1242 child
1243 .as_any()
1244 .downcast_ref::<PrimitiveArray<T>>()
1245 .ok_or_else(|| {
1246 DataFusionError::Execution(format!(
1247 "native histogram field {name} has invalid type {}",
1248 child.data_type()
1249 ))
1250 })
1251}
1252
1253fn list_child<'a>(array: &'a StructArray, name: &str) -> DfResult<&'a ListArray> {
1254 let child = struct_child(array, name)?;
1255 child.as_any().downcast_ref::<ListArray>().ok_or_else(|| {
1256 DataFusionError::Execution(format!(
1257 "native histogram field {name} has invalid type {}",
1258 child.data_type()
1259 ))
1260 })
1261}
1262
1263fn required_primitive<T>(array: &StructArray, name: &str, row: usize) -> DfResult<T::Native>
1264where
1265 T: ArrowPrimitiveType,
1266{
1267 let values = primitive_child::<T>(array, name)?;
1268 if values.is_null(row) {
1269 return Err(DataFusionError::Execution(format!(
1270 "native histogram field {name} is null"
1271 )));
1272 }
1273 Ok(values.value(row))
1274}
1275
1276fn optional_primitive<T>(array: &StructArray, name: &str, row: usize) -> DfResult<Option<T::Native>>
1277where
1278 T: ArrowPrimitiveType,
1279{
1280 let values = primitive_child::<T>(array, name)?;
1281 Ok((!values.is_null(row)).then(|| values.value(row)))
1282}
1283
1284fn list_values<T>(array: &StructArray, name: &str, row: usize) -> DfResult<Vec<T::Native>>
1285where
1286 T: ArrowPrimitiveType,
1287{
1288 let list = list_child(array, name)?;
1289 if list.is_null(row) {
1290 return Ok(Vec::new());
1291 }
1292
1293 let values = list.value(row);
1294 let values = values
1295 .as_any()
1296 .downcast_ref::<PrimitiveArray<T>>()
1297 .ok_or_else(|| {
1298 DataFusionError::Execution(format!(
1299 "native histogram list field {name} has invalid value type {}",
1300 values.data_type()
1301 ))
1302 })?;
1303
1304 values
1305 .iter()
1306 .map(|value| {
1307 value.ok_or_else(|| {
1308 DataFusionError::Execution(format!(
1309 "native histogram list field {name} contains null"
1310 ))
1311 })
1312 })
1313 .collect()
1314}
1315
1316fn read_spans(offsets: Vec<i32>, lengths: Vec<u32>, name: &str) -> DfResult<Vec<Span>> {
1317 if offsets.len() != lengths.len() {
1318 return Err(DataFusionError::Execution(format!(
1319 "native histogram {name} span offsets and lengths mismatch: {} vs {}",
1320 offsets.len(),
1321 lengths.len()
1322 )));
1323 }
1324 Ok(offsets
1325 .into_iter()
1326 .zip(lengths)
1327 .map(|(offset, length)| Span { offset, length })
1328 .collect())
1329}
1330
1331fn check_span_bucket_count(spans: &[Span], buckets: usize, name: &str) -> DfResult<()> {
1332 let span_len = span_bucket_len(spans).ok_or_else(|| {
1333 DataFusionError::Execution(format!("native histogram {name} spans overflow"))
1334 })?;
1335 if span_len != buckets {
1336 return Err(DataFusionError::Execution(format!(
1337 "native histogram {name} spans describe {span_len} buckets, found {buckets}"
1338 )));
1339 }
1340 Ok(())
1341}
1342
1343pub fn read_histogram(array: &StructArray, row: usize) -> DfResult<Option<NativeHistogram>> {
1348 if array.is_null(row) {
1349 return Ok(None);
1350 }
1351
1352 let schema = required_primitive::<Int32Type>(array, SCHEMA_FIELD, row)?;
1353 let positive_spans = read_spans(
1354 list_values::<Int32Type>(array, POSITIVE_SPAN_OFFSETS_FIELD, row)?,
1355 list_values::<UInt32Type>(array, POSITIVE_SPAN_LENGTHS_FIELD, row)?,
1356 "positive",
1357 )?;
1358 let negative_spans = read_spans(
1359 list_values::<Int32Type>(array, NEGATIVE_SPAN_OFFSETS_FIELD, row)?,
1360 list_values::<UInt32Type>(array, NEGATIVE_SPAN_LENGTHS_FIELD, row)?,
1361 "negative",
1362 )?;
1363
1364 let (count, zero_count, positive_buckets, negative_buckets) =
1365 if let Some(count) = optional_primitive::<Float64Type>(array, COUNT_F64_FIELD, row)? {
1366 (
1367 count,
1368 optional_primitive::<Float64Type>(array, ZERO_COUNT_F64_FIELD, row)?
1369 .unwrap_or_default(),
1370 list_values::<Float64Type>(array, POSITIVE_BUCKETS_F64_FIELD, row)?,
1371 list_values::<Float64Type>(array, NEGATIVE_BUCKETS_F64_FIELD, row)?,
1372 )
1373 } else {
1374 (
1375 required_primitive::<UInt64Type>(array, COUNT_U64_FIELD, row)? as f64,
1376 optional_primitive::<UInt64Type>(array, ZERO_COUNT_U64_FIELD, row)?
1377 .unwrap_or_default() as f64,
1378 list_values::<Int64Type>(array, POSITIVE_BUCKETS_I64_FIELD, row)?
1379 .into_iter()
1380 .map(|value| value as f64)
1381 .collect(),
1382 list_values::<Int64Type>(array, NEGATIVE_BUCKETS_I64_FIELD, row)?
1383 .into_iter()
1384 .map(|value| value as f64)
1385 .collect(),
1386 )
1387 };
1388
1389 check_span_bucket_count(&positive_spans, positive_buckets.len(), "positive")?;
1390 check_span_bucket_count(&negative_spans, negative_buckets.len(), "negative")?;
1391
1392 Ok(Some(NativeHistogram {
1393 schema,
1394 zero_threshold: required_primitive::<Float64Type>(array, ZERO_THRESHOLD_FIELD, row)?,
1395 sum: required_primitive::<Float64Type>(array, SUM_FIELD, row)?,
1396 reset_hint: required_primitive::<Int32Type>(array, RESET_HINT_FIELD, row)?.into(),
1397 start_timestamp: optional_primitive::<TimestampMillisecondType>(
1398 array,
1399 START_TIMESTAMP_FIELD,
1400 row,
1401 )?,
1402 custom_values: list_values::<Float64Type>(array, CUSTOM_VALUES_FIELD, row)?,
1403 positive_spans,
1404 negative_spans,
1405 count,
1406 zero_count,
1407 positive_buckets,
1408 negative_buckets,
1409 }))
1410}
1411
1412fn list_opt<T>(values: Vec<T>) -> Option<Vec<Option<T>>> {
1413 Some(values.into_iter().map(Some).collect())
1414}
1415
1416pub fn build_histogram_array(values: &[Option<NativeHistogram>]) -> ArrayRef {
1422 let mut schemas = Vec::with_capacity(values.len());
1423 let mut zero_thresholds = Vec::with_capacity(values.len());
1424 let mut sums = Vec::with_capacity(values.len());
1425 let mut reset_hints = Vec::with_capacity(values.len());
1426 let mut start_timestamps = Vec::with_capacity(values.len());
1427 let mut custom_values = Vec::with_capacity(values.len());
1428 let mut positive_span_offsets = Vec::with_capacity(values.len());
1429 let mut positive_span_lengths = Vec::with_capacity(values.len());
1430 let mut negative_span_offsets = Vec::with_capacity(values.len());
1431 let mut negative_span_lengths = Vec::with_capacity(values.len());
1432 let mut count_u64 = Vec::<Option<u64>>::with_capacity(values.len());
1433 let mut zero_count_u64 = Vec::<Option<u64>>::with_capacity(values.len());
1434 let mut positive_buckets_i64 = Vec::with_capacity(values.len());
1435 let mut negative_buckets_i64 = Vec::with_capacity(values.len());
1436 let mut count_f64 = Vec::with_capacity(values.len());
1437 let mut zero_count_f64 = Vec::with_capacity(values.len());
1438 let mut positive_buckets_f64 = Vec::with_capacity(values.len());
1439 let mut negative_buckets_f64 = Vec::with_capacity(values.len());
1440 let mut validity = Vec::with_capacity(values.len());
1441
1442 for value in values {
1443 validity.push(value.is_some());
1444 if let Some(histogram) = value {
1445 schemas.push(Some(histogram.schema));
1446 zero_thresholds.push(Some(histogram.zero_threshold));
1447 sums.push(Some(histogram.sum));
1448 reset_hints.push(Some(i32::from(histogram.reset_hint)));
1449 start_timestamps.push(histogram.start_timestamp);
1450 custom_values.push(list_opt(histogram.custom_values.clone()));
1451 positive_span_offsets.push(list_opt(
1452 histogram
1453 .positive_spans
1454 .iter()
1455 .map(|span| span.offset)
1456 .collect(),
1457 ));
1458 positive_span_lengths.push(list_opt(
1459 histogram
1460 .positive_spans
1461 .iter()
1462 .map(|span| span.length)
1463 .collect(),
1464 ));
1465 negative_span_offsets.push(list_opt(
1466 histogram
1467 .negative_spans
1468 .iter()
1469 .map(|span| span.offset)
1470 .collect(),
1471 ));
1472 negative_span_lengths.push(list_opt(
1473 histogram
1474 .negative_spans
1475 .iter()
1476 .map(|span| span.length)
1477 .collect(),
1478 ));
1479 count_u64.push(None);
1480 zero_count_u64.push(None);
1481 positive_buckets_i64.push(list_opt(Vec::<i64>::new()));
1482 negative_buckets_i64.push(list_opt(Vec::<i64>::new()));
1483 count_f64.push(Some(histogram.count));
1484 zero_count_f64.push(Some(histogram.zero_count));
1485 positive_buckets_f64.push(list_opt(histogram.positive_buckets.clone()));
1486 negative_buckets_f64.push(list_opt(histogram.negative_buckets.clone()));
1487 } else {
1488 schemas.push(None);
1489 zero_thresholds.push(None);
1490 sums.push(None);
1491 reset_hints.push(None);
1492 start_timestamps.push(None);
1493 custom_values.push(None);
1494 positive_span_offsets.push(None);
1495 positive_span_lengths.push(None);
1496 negative_span_offsets.push(None);
1497 negative_span_lengths.push(None);
1498 count_u64.push(None);
1499 zero_count_u64.push(None);
1500 positive_buckets_i64.push(None);
1501 negative_buckets_i64.push(None);
1502 count_f64.push(None);
1503 zero_count_f64.push(None);
1504 positive_buckets_f64.push(None);
1505 negative_buckets_f64.push(None);
1506 }
1507 }
1508
1509 let named_arrays: Vec<(&str, ArrayRef)> = vec![
1510 (SCHEMA_FIELD, Arc::new(Int32Array::from(schemas))),
1511 (
1512 ZERO_THRESHOLD_FIELD,
1513 Arc::new(Float64Array::from(zero_thresholds)),
1514 ),
1515 (SUM_FIELD, Arc::new(Float64Array::from(sums))),
1516 (RESET_HINT_FIELD, Arc::new(Int32Array::from(reset_hints))),
1517 (
1518 START_TIMESTAMP_FIELD,
1519 Arc::new(TimestampMillisecondArray::from_iter(start_timestamps)),
1520 ),
1521 (
1522 CUSTOM_VALUES_FIELD,
1523 Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(
1524 custom_values,
1525 )),
1526 ),
1527 (
1528 POSITIVE_SPAN_OFFSETS_FIELD,
1529 Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
1530 positive_span_offsets,
1531 )),
1532 ),
1533 (
1534 POSITIVE_SPAN_LENGTHS_FIELD,
1535 Arc::new(ListArray::from_iter_primitive::<UInt32Type, _, _>(
1536 positive_span_lengths,
1537 )),
1538 ),
1539 (
1540 NEGATIVE_SPAN_OFFSETS_FIELD,
1541 Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
1542 negative_span_offsets,
1543 )),
1544 ),
1545 (
1546 NEGATIVE_SPAN_LENGTHS_FIELD,
1547 Arc::new(ListArray::from_iter_primitive::<UInt32Type, _, _>(
1548 negative_span_lengths,
1549 )),
1550 ),
1551 (COUNT_U64_FIELD, Arc::new(UInt64Array::from(count_u64))),
1552 (
1553 ZERO_COUNT_U64_FIELD,
1554 Arc::new(UInt64Array::from(zero_count_u64)),
1555 ),
1556 (
1557 POSITIVE_BUCKETS_I64_FIELD,
1558 Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(
1559 positive_buckets_i64,
1560 )),
1561 ),
1562 (
1563 NEGATIVE_BUCKETS_I64_FIELD,
1564 Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(
1565 negative_buckets_i64,
1566 )),
1567 ),
1568 (COUNT_F64_FIELD, Arc::new(Float64Array::from(count_f64))),
1569 (
1570 ZERO_COUNT_F64_FIELD,
1571 Arc::new(Float64Array::from(zero_count_f64)),
1572 ),
1573 (
1574 POSITIVE_BUCKETS_F64_FIELD,
1575 Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(
1576 positive_buckets_f64,
1577 )),
1578 ),
1579 (
1580 NEGATIVE_BUCKETS_F64_FIELD,
1581 Arc::new(ListArray::from_iter_primitive::<Float64Type, _, _>(
1582 negative_buckets_f64,
1583 )),
1584 ),
1585 ];
1586 let (fields, arrays): (Vec<_>, Vec<_>) = named_arrays
1587 .into_iter()
1588 .map(|(name, array)| (Field::new(name, array.data_type().clone(), true), array))
1589 .unzip();
1590
1591 Arc::new(StructArray::new(
1592 fields.into(),
1593 arrays,
1594 Some(NullBuffer::from(validity)),
1595 ))
1596}
1597
1598#[cfg(test)]
1599mod tests {
1600 use super::*;
1601
1602 fn histogram(positive_spans: Vec<Span>, positive_buckets: Vec<f64>) -> NativeHistogram {
1603 let count = positive_buckets.iter().sum();
1604 NativeHistogram {
1605 schema: 0,
1606 zero_threshold: 0.0,
1607 sum: count,
1608 reset_hint: CounterResetHint::CounterReset,
1609 start_timestamp: None,
1610 custom_values: Vec::new(),
1611 positive_spans,
1612 negative_spans: Vec::new(),
1613 count,
1614 zero_count: 0.0,
1615 positive_buckets,
1616 negative_buckets: Vec::new(),
1617 }
1618 }
1619
1620 #[test]
1621 fn span_rebuild_checks_offsets_and_preserves_empty_input() {
1622 assert_eq!(
1623 spans_from_indices_counts(Vec::new()),
1624 Some((Vec::new(), Vec::new()))
1625 );
1626 assert_eq!(
1627 spans_from_indices_counts(vec![(2, 1.0), (3, 2.0), (5, 3.0)]),
1628 Some((
1629 vec![
1630 Span {
1631 offset: 2,
1632 length: 2,
1633 },
1634 Span {
1635 offset: 1,
1636 length: 1,
1637 },
1638 ],
1639 vec![1.0, 2.0, 3.0],
1640 ))
1641 );
1642 assert_eq!(
1643 spans_from_indices_counts(vec![(i32::MIN, 1.0), (i32::MAX, 2.0)]),
1644 None
1645 );
1646 }
1647
1648 #[test]
1649 fn arrow_round_trip_preserves_unrecognized_reset_hint() {
1650 let mut expected = histogram(
1651 vec![Span {
1652 offset: 0,
1653 length: 1,
1654 }],
1655 vec![1.0],
1656 );
1657 expected.reset_hint = CounterResetHint::Unrecognized(42);
1658
1659 let array = build_histogram_array(&[Some(expected.clone())]);
1660 assert_eq!(array.data_type(), &native_histogram_arrow_type());
1661 let array = array.as_any().downcast_ref::<StructArray>().unwrap();
1662
1663 assert_eq!(read_histogram(array, 0).unwrap(), Some(expected));
1664 assert!(
1665 primitive_child::<UInt64Type>(array, COUNT_U64_FIELD)
1666 .unwrap()
1667 .is_null(0)
1668 );
1669 assert_eq!(
1670 primitive_child::<Float64Type>(array, COUNT_F64_FIELD)
1671 .unwrap()
1672 .value(0),
1673 1.0
1674 );
1675 }
1676
1677 #[test]
1678 fn add_uses_sparse_bucket_union() {
1679 let left = histogram(
1680 vec![Span {
1681 offset: 0,
1682 length: 1,
1683 }],
1684 vec![1.0],
1685 );
1686 let right = histogram(
1687 vec![Span {
1688 offset: 2,
1689 length: 1,
1690 }],
1691 vec![2.0],
1692 );
1693
1694 let result = left.add(&right).unwrap();
1695 assert_eq!(
1696 result.positive_spans,
1697 vec![
1698 Span {
1699 offset: 0,
1700 length: 1,
1701 },
1702 Span {
1703 offset: 1,
1704 length: 1,
1705 },
1706 ]
1707 );
1708 assert_eq!(result.positive_buckets, vec![1.0, 2.0]);
1709 }
1710
1711 #[test]
1712 fn add_combines_reset_hints_for_compatible_empty_payloads() {
1713 let mut empty = histogram(Vec::new(), Vec::new());
1714 empty.reset_hint = CounterResetHint::Gauge;
1715 let mut populated = histogram(
1716 vec![Span {
1717 offset: 0,
1718 length: 1,
1719 }],
1720 vec![1.0],
1721 );
1722 populated.reset_hint = CounterResetHint::NotCounterReset;
1723
1724 assert_eq!(
1725 empty.add(&populated).unwrap().reset_hint,
1726 CounterResetHint::Gauge
1727 );
1728 assert_eq!(
1729 populated.add(&empty).unwrap().reset_hint,
1730 CounterResetHint::Gauge
1731 );
1732 }
1733
1734 #[test]
1735 fn empty_arithmetic_rejects_incompatible_schemas() {
1736 let exponential = histogram(Vec::new(), Vec::new());
1737 let mut custom = histogram(
1738 vec![Span {
1739 offset: 0,
1740 length: 1,
1741 }],
1742 vec![2.0],
1743 );
1744 custom.schema = CUSTOM_BUCKETS_SCHEMA;
1745 custom.custom_values = vec![1.0];
1746
1747 assert!(exponential.add(&custom).is_none());
1748 assert!(custom.add(&exponential).is_none());
1749 assert!(exponential.sub(&custom).is_none());
1750 assert!(custom.sub(&exponential).is_none());
1751 }
1752
1753 #[test]
1754 fn empty_custom_histogram_still_reconciles_bounds() {
1755 let mut empty = histogram(Vec::new(), Vec::new());
1756 empty.schema = CUSTOM_BUCKETS_SCHEMA;
1757 empty.custom_values = vec![1.0];
1758 let mut populated = histogram(
1759 vec![Span {
1760 offset: 0,
1761 length: 1,
1762 }],
1763 vec![2.0],
1764 );
1765 populated.schema = CUSTOM_BUCKETS_SCHEMA;
1766 populated.custom_values = vec![2.0];
1767
1768 let result = empty.add(&populated).unwrap();
1769 assert!(result.custom_values.is_empty());
1770 assert_eq!(result.positive_buckets, vec![2.0]);
1771 }
1772
1773 #[test]
1774 fn zero_threshold_expands_to_populated_bucket_boundary() {
1775 let left = NativeHistogram {
1776 zero_threshold: 0.5,
1777 ..histogram(
1778 vec![Span {
1779 offset: 0,
1780 length: 1,
1781 }],
1782 vec![1.0],
1783 )
1784 };
1785 let right = NativeHistogram {
1786 zero_threshold: 0.75,
1787 ..histogram(
1788 vec![Span {
1789 offset: 1,
1790 length: 1,
1791 }],
1792 vec![1.0],
1793 )
1794 };
1795
1796 let result = left.add(&right).unwrap();
1797 assert_eq!(result.zero_threshold, 1.0);
1798 assert_eq!(result.zero_count, 1.0);
1799 assert_eq!(result.positive_buckets, vec![1.0]);
1800 }
1801
1802 #[test]
1803 fn custom_reconciliation_without_shared_bounds_uses_overflow_bucket() {
1804 let mut left = histogram(
1805 vec![Span {
1806 offset: 0,
1807 length: 1,
1808 }],
1809 vec![1.0],
1810 );
1811 left.schema = CUSTOM_BUCKETS_SCHEMA;
1812 left.custom_values = vec![1.0];
1813
1814 let mut right = histogram(
1815 vec![Span {
1816 offset: 0,
1817 length: 1,
1818 }],
1819 vec![2.0],
1820 );
1821 right.schema = CUSTOM_BUCKETS_SCHEMA;
1822 right.custom_values = vec![2.0];
1823
1824 let result = left.add(&right).unwrap();
1825 assert!(result.custom_values.is_empty());
1826 assert_eq!(
1827 result.positive_spans,
1828 vec![Span {
1829 offset: 0,
1830 length: 1,
1831 }]
1832 );
1833 assert_eq!(result.positive_buckets, vec![3.0]);
1834 }
1835
1836 #[test]
1837 fn promql_eq_ignores_metadata_but_compares_sparse_layout() {
1838 let mut left = histogram(
1839 vec![Span {
1840 offset: 0,
1841 length: 2,
1842 }],
1843 vec![1.0, 0.0],
1844 );
1845 left.reset_hint = CounterResetHint::CounterReset;
1846 left.start_timestamp = Some(1000);
1847
1848 let mut right = histogram(
1849 vec![Span {
1850 offset: 0,
1851 length: 1,
1852 }],
1853 vec![1.0],
1854 );
1855 right.reset_hint = CounterResetHint::NotCounterReset;
1856 right.start_timestamp = Some(2000);
1857
1858 assert!(!left.promql_eq(&right));
1859
1860 right.positive_spans = vec![
1861 Span {
1862 offset: 0,
1863 length: 0,
1864 },
1865 Span {
1866 offset: 0,
1867 length: 1,
1868 },
1869 Span {
1870 offset: 42,
1871 length: 0,
1872 },
1873 ];
1874 left.positive_spans = vec![Span {
1875 offset: 0,
1876 length: 1,
1877 }];
1878 left.positive_buckets = vec![1.0];
1879 assert!(left.promql_eq(&right));
1880 }
1881
1882 #[test]
1883 fn promql_eq_compares_payload_floats_by_bits() {
1884 let nan = f64::from_bits(0x7ff8_0000_0000_0001);
1885 let other_nan = f64::from_bits(0x7ff8_0000_0000_0002);
1886 let mut left = histogram(
1887 vec![Span {
1888 offset: 0,
1889 length: 1,
1890 }],
1891 vec![nan],
1892 );
1893 left.count = nan;
1894 left.zero_count = nan;
1895 left.sum = nan;
1896 let right = left.clone();
1897
1898 assert!(left.promql_eq(&right));
1899
1900 for changed in [
1901 NativeHistogram {
1902 count: other_nan,
1903 ..right.clone()
1904 },
1905 NativeHistogram {
1906 zero_count: other_nan,
1907 ..right.clone()
1908 },
1909 NativeHistogram {
1910 sum: other_nan,
1911 ..right.clone()
1912 },
1913 NativeHistogram {
1914 positive_buckets: vec![other_nan],
1915 ..right.clone()
1916 },
1917 ] {
1918 assert!(!left.promql_eq(&changed));
1919 }
1920
1921 let undecodable = histogram(
1922 vec![Span {
1923 offset: i32::MAX,
1924 length: 1,
1925 }],
1926 vec![1.0],
1927 );
1928 assert!(!undecodable.promql_eq(&undecodable.clone()));
1929
1930 let mut mismatched = histogram(
1931 vec![Span {
1932 offset: 0,
1933 length: 2,
1934 }],
1935 vec![1.0],
1936 );
1937 mismatched.count = 1.0;
1938 mismatched.sum = 1.0;
1939 assert!(!mismatched.promql_eq(&mismatched.clone()));
1940
1941 let invalid_offset = histogram(
1942 vec![
1943 Span {
1944 offset: 0,
1945 length: 1,
1946 },
1947 Span {
1948 offset: -1,
1949 length: 1,
1950 },
1951 ],
1952 vec![1.0, 1.0],
1953 );
1954 assert!(!invalid_offset.promql_eq(&invalid_offset.clone()));
1955 }
1956
1957 #[test]
1958 fn detect_reset_preserves_layout_direction() {
1959 let mut previous = histogram(
1960 vec![Span {
1961 offset: 1,
1962 length: 1,
1963 }],
1964 vec![1.0],
1965 );
1966 previous.reset_hint = CounterResetHint::Unknown;
1967
1968 let mut higher_resolution = previous.clone();
1969 higher_resolution.schema = 1;
1970 higher_resolution.positive_spans[0].offset = 2;
1971 assert!(higher_resolution.detect_reset(&previous));
1972
1973 let mut lower_resolution = previous.clone();
1974 lower_resolution.schema = 0;
1975 lower_resolution.positive_spans[0].offset = 1;
1976 let mut previous_higher_resolution = higher_resolution;
1977 previous_higher_resolution.reset_hint = CounterResetHint::Unknown;
1978 assert!(!lower_resolution.detect_reset(&previous_higher_resolution));
1979
1980 let mut smaller_zero_threshold = previous.clone();
1981 smaller_zero_threshold.zero_threshold = 0.5;
1982 previous.zero_threshold = 1.0;
1983 assert!(smaller_zero_threshold.detect_reset(&previous));
1984
1985 let mut previous = histogram(
1986 vec![Span {
1987 offset: 0,
1988 length: 1,
1989 }],
1990 vec![1.0],
1991 );
1992 previous.reset_hint = CounterResetHint::Unknown;
1993 let mut split_bucket = histogram(Vec::new(), Vec::new());
1994 split_bucket.reset_hint = CounterResetHint::Unknown;
1995 split_bucket.count = 1.0;
1996 split_bucket.zero_count = 1.0;
1997 split_bucket.zero_threshold = 0.75;
1998 assert!(split_bucket.detect_reset(&previous));
1999
2000 split_bucket.zero_threshold = 1.0;
2001 assert!(!split_bucket.detect_reset(&previous));
2002 }
2003
2004 #[test]
2005 fn extreme_bucket_index_returns_none_instead_of_overflowing() {
2006 assert_eq!(ceil_div(i32::MIN, 2), i32::MIN / 2);
2007
2008 let mut histogram = histogram(
2009 vec![Span {
2010 offset: i32::MIN,
2011 length: 1,
2012 }],
2013 vec![1.0],
2014 );
2015 assert!(histogram.side_buckets(true).is_none());
2016
2017 histogram.schema = 1;
2018 let reduced = histogram.copy_to_schema(0).unwrap();
2019 assert_eq!(reduced.positive_spans[0].offset, i32::MIN / 2);
2020 }
2021
2022 #[test]
2023 fn exponential_bucket_bounds_stop_after_overflow_bucket() {
2024 for schema in [-4, 0, 8] {
2025 let overflow = exponential_overflow_bucket_index(schema).unwrap();
2026 assert_eq!(get_bound(overflow - 1, schema, &[]), Some(f64::MAX));
2027 assert_eq!(get_bound(overflow, schema, &[]), Some(f64::INFINITY));
2028 assert_eq!(get_bound(overflow + 1, schema, &[]), None);
2029 }
2030 assert_eq!(exponential_overflow_bucket_index(-5), None);
2031 assert_eq!(exponential_overflow_bucket_index(9), None);
2032 assert_eq!(get_bound(i32::MIN, -4, &[]), Some(0.0));
2033 }
2034
2035 #[test]
2036 fn custom_bucket_midpoints_preserve_infinite_bounds() {
2037 let mut histogram = histogram(Vec::new(), Vec::new());
2038 histogram.schema = CUSTOM_BUCKETS_SCHEMA;
2039
2040 assert_eq!(
2041 histogram.bucket_midpoint(&Bucket {
2042 lower: f64::NEG_INFINITY,
2043 upper: -1.0,
2044 count: 1.0,
2045 boundary_rule: BoundaryRule::OpenLeft,
2046 }),
2047 f64::NEG_INFINITY
2048 );
2049 assert_eq!(
2050 histogram.bucket_midpoint(&Bucket {
2051 lower: 1.0,
2052 upper: f64::INFINITY,
2053 count: 1.0,
2054 boundary_rule: BoundaryRule::OpenLeft,
2055 }),
2056 f64::INFINITY
2057 );
2058 assert!(
2059 histogram
2060 .bucket_midpoint(&Bucket {
2061 lower: f64::NEG_INFINITY,
2062 upper: f64::INFINITY,
2063 count: 1.0,
2064 boundary_rule: BoundaryRule::OpenLeft,
2065 })
2066 .is_nan()
2067 );
2068 }
2069
2070 #[test]
2071 fn fraction_excludes_nan_observations() {
2072 let mut histogram = histogram(
2073 vec![Span {
2074 offset: 0,
2075 length: 1,
2076 }],
2077 vec![8.0],
2078 );
2079 histogram.count = 10.0;
2080 histogram.sum = f64::NAN;
2081
2082 assert_eq!(histogram.fraction(f64::NEG_INFINITY, f64::INFINITY), 0.8);
2083 }
2084
2085 #[test]
2086 fn subtraction_returns_gauge_histogram() {
2087 let left = histogram(
2088 vec![Span {
2089 offset: 0,
2090 length: 1,
2091 }],
2092 vec![3.0],
2093 );
2094 let right = histogram(
2095 vec![Span {
2096 offset: 0,
2097 length: 1,
2098 }],
2099 vec![1.0],
2100 );
2101
2102 let result = left.sub(&right).unwrap();
2103 assert_eq!(result.reset_hint, CounterResetHint::Gauge);
2104 assert_eq!(result.positive_buckets, vec![2.0]);
2105 }
2106
2107 #[test]
2108 fn subtraction_treats_compatible_empty_left_as_zero() {
2109 let left = histogram(vec![], vec![]);
2110 let right = histogram(
2111 vec![Span {
2112 offset: 0,
2113 length: 1,
2114 }],
2115 vec![2.0],
2116 );
2117
2118 let result = left.sub(&right).unwrap();
2119 assert_eq!(result.reset_hint, CounterResetHint::Gauge);
2120 assert_eq!(result.count, -2.0);
2121 assert_eq!(result.sum, -2.0);
2122 assert_eq!(result.positive_buckets, vec![-2.0]);
2123 }
2124
2125 #[test]
2126 fn subtraction_rejects_incompatible_empty_left() {
2127 let left = histogram(vec![], vec![]);
2128 let mut right = histogram(
2129 vec![Span {
2130 offset: 0,
2131 length: 1,
2132 }],
2133 vec![2.0],
2134 );
2135 right.schema = CUSTOM_BUCKETS_SCHEMA;
2136 right.custom_values = vec![1.0];
2137
2138 assert!(left.sub(&right).is_none());
2139 }
2140}
2141
2142pub const NATIVE_HISTOGRAM_SUBFIELD_ID_BASE: i32 = 0x5000_0000;
2162
2163pub const NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE: i32 = 64;
2166
2167pub const NATIVE_HISTOGRAM_LIST_ELEMENT_OFFSET: i32 = 32;
2169
2170pub fn native_histogram_subfield_id(column_id: i32, name: &str) -> Option<i32> {
2174 let idx = subfield_index(name)?;
2175 NATIVE_HISTOGRAM_SUBFIELD_ID_BASE
2176 .checked_add(column_id.checked_mul(NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE)?)
2177 .and_then(|v| v.checked_add(idx))
2178}
2179
2180pub fn native_histogram_list_element_id(column_id: i32, name: &str) -> Option<i32> {
2184 let idx = subfield_index(name)?;
2185 NATIVE_HISTOGRAM_SUBFIELD_ID_BASE
2186 .checked_add(column_id.checked_mul(NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE)?)
2187 .and_then(|v| v.checked_add(NATIVE_HISTOGRAM_LIST_ELEMENT_OFFSET))
2188 .and_then(|v| v.checked_add(idx))
2189}
2190
2191fn subfield_index(name: &str) -> Option<i32> {
2192 NATIVE_HISTOGRAM_FIELD_NAMES
2193 .iter()
2194 .position(|n| *n == name)
2195 .map(|i| i as i32)
2196}
2197
2198#[cfg(test)]
2199mod subfield_id_tests {
2200 use super::*;
2201
2202 #[test]
2203 fn subfield_ids_are_namespaced_and_disjoint() {
2204 let sum_idx = 2;
2206 let custom_values_idx = 5;
2207
2208 assert_eq!(
2211 native_histogram_subfield_id(1, SUM_FIELD),
2212 Some(NATIVE_HISTOGRAM_SUBFIELD_ID_BASE + NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE + sum_idx,)
2213 );
2214 assert_eq!(
2216 native_histogram_list_element_id(1, CUSTOM_VALUES_FIELD),
2217 Some(
2218 NATIVE_HISTOGRAM_SUBFIELD_ID_BASE
2219 + NATIVE_HISTOGRAM_SUBFIELD_ID_STRIDE
2220 + NATIVE_HISTOGRAM_LIST_ELEMENT_OFFSET
2221 + custom_values_idx,
2222 )
2223 );
2224 assert_ne!(
2226 native_histogram_subfield_id(1, SUM_FIELD),
2227 native_histogram_subfield_id(7, SUM_FIELD)
2228 );
2229 assert_eq!(native_histogram_subfield_id(1, "not_a_field"), None);
2231 }
2232
2233 #[test]
2234 fn subfield_ids_overflow_returns_none() {
2235 assert_eq!(native_histogram_subfield_id(12_582_912, SUM_FIELD), None);
2240 assert_eq!(
2241 native_histogram_list_element_id(12_582_912, CUSTOM_VALUES_FIELD),
2242 None
2243 );
2244 assert!(native_histogram_subfield_id(12_582_911, SUM_FIELD).is_some());
2246 }
2247}