diff --git a/src/promql/src/range_eval/function.rs b/src/promql/src/range_eval/function.rs index a8c1aae085b..ce54432ed28 100644 --- a/src/promql/src/range_eval/function.rs +++ b/src/promql/src/range_eval/function.rs @@ -420,8 +420,14 @@ fn extrapolated_rate_of( } let zero_point = (is_counter && result > 0.0 && first >= 0.0).then(|| first / result); let factor = extrapolation_factor( - (first_t, last_t), - samples_minus_one, + ExtrapolationSpan { + // A float sample carries no start timestamp, so both intervals start at the first + // sample. + sampled_from: first_t, + average_from: first_t, + last: last_t, + samples_minus_one, + }, eval_ts, range_ms, is_counter, @@ -464,27 +470,64 @@ pub(crate) fn extrapolated_rate_batch, ) -> f64 { + let ExtrapolationSpan { + sampled_from, + average_from, + last, + samples_minus_one, + } = span; // Millisecond differences in 64-bit arithmetic: widening to `i128` and narrowing back with // `as f64` compiles to a call to the `__floattidf` builtin instead of an instruction, once per // grid point per series. The subtractions wrap, which is what Go's `int64` arithmetic does, so // an out-of-range window start produces the same value Prometheus computes rather than a // clamped one; real timestamps are nowhere near that range, where both agree. let range_start = eval_ts.wrapping_sub(range_ms); - let mut duration_to_start = first_t.wrapping_sub(range_start) as f64 / 1000.0; - let mut duration_to_end = eval_ts.wrapping_sub(last_t) as f64 / 1000.0; - let sampled_interval = last_t.wrapping_sub(first_t) as f64 / 1000.0; - let average_interval = sampled_interval / samples_minus_one as f64; + let mut duration_to_start = average_from.wrapping_sub(range_start) as f64 / 1000.0; + let mut duration_to_end = eval_ts.wrapping_sub(last) as f64 / 1000.0; + let sampled_interval = last.wrapping_sub(sampled_from) as f64 / 1000.0; + let average_interval = if samples_minus_one == 0 { + 0.0 + } else { + last.wrapping_sub(average_from) as f64 / 1000.0 / samples_minus_one as f64 + }; let threshold = average_interval * 1.1; if duration_to_start >= threshold { duration_to_start = average_interval / 2.0; @@ -645,3 +688,268 @@ impl std::hash::Hash for RangeFunction { } } } + +#[cfg(test)] +mod tests { + use super::{ExtrapolationSpan, extrapolation_factor}; + + /// One window as both range-function routes read it: `sampled_from` starts the interval the + /// increase was sampled over, `average_from` the interval the average sample distance is + /// measured over. They are the same timestamp except for a native histogram counter with a + /// synthetic zero, whose sampled interval starts at the zero. + #[derive(Clone, Copy, Debug)] + struct Window { + sampled_from: i64, + average_from: i64, + last: i64, + samples_minus_one: usize, + eval_ts: i64, + range_ms: i64, + is_counter: bool, + is_rate: bool, + zero_point: Option, + } + + impl Window { + /// The factor the float route computes. A float sample carries no start timestamp, so its + /// single anchor starts both intervals and `sampled_from` plays no part. + fn as_float(self) -> f64 { + extrapolation_factor( + ExtrapolationSpan { + sampled_from: self.average_from, + average_from: self.average_from, + last: self.last, + samples_minus_one: self.samples_minus_one, + }, + self.eval_ts, + self.range_ms, + self.is_counter, + self.is_rate, + self.zero_point, + ) + } + + /// The factor the native histogram route computes, with both anchors as its call site + /// builds them. + fn as_histogram(self) -> f64 { + extrapolation_factor( + ExtrapolationSpan { + sampled_from: self.sampled_from, + average_from: self.average_from, + last: self.last, + samples_minus_one: self.samples_minus_one, + }, + self.eval_ts, + self.range_ms, + self.is_counter, + self.is_rate, + self.zero_point, + ) + } + } + + /// Both routes reach one function, so their factors can only differ in how each fills the span. + /// A window without a synthetic zero is filled the same way by both, and the factors have to + /// agree to the bit. A window whose sampled anchor is a synthetic zero is the one shape the two + /// fills answer differently, and pinning both answers there is what puts the anchors, not only + /// the arithmetic, under the assertions. + #[test] + fn both_routes_compute_one_extrapolation_factor() { + // Evaluation at 6500ms over a 6000ms range: the window opens at 500ms and the last sample + // sits 1500ms inside its closed end. + let windows = [ + Window { + sampled_from: 2_000, + average_from: 2_000, + last: 5_000, + samples_minus_one: 1, + eval_ts: 6_500, + range_ms: 6_000, + is_counter: true, + is_rate: false, + zero_point: None, + }, + Window { + sampled_from: 2_000, + average_from: 2_000, + last: 5_000, + samples_minus_one: 2, + eval_ts: 6_500, + range_ms: 6_000, + is_counter: true, + is_rate: true, + zero_point: None, + }, + Window { + sampled_from: 2_000, + average_from: 2_000, + last: 5_000, + samples_minus_one: 2, + eval_ts: 6_500, + range_ms: 6_000, + is_counter: false, + is_rate: false, + zero_point: None, + }, + // The last sample further in than the average interval allows, so both ends fall back + // to half of it. + Window { + sampled_from: 4_000, + average_from: 4_000, + last: 4_500, + samples_minus_one: 1, + eval_ts: 6_500, + range_ms: 6_000, + is_counter: true, + is_rate: true, + zero_point: None, + }, + // A single sample holds no average interval to measure. + Window { + sampled_from: 5_000, + average_from: 5_000, + last: 5_000, + samples_minus_one: 0, + eval_ts: 6_500, + range_ms: 6_000, + is_counter: true, + is_rate: false, + zero_point: None, + }, + ]; + for window in windows { + for zero_point in [None, Some(0.0), Some(0.25), Some(1.5)] { + let window = Window { + zero_point, + ..window + }; + assert_eq!( + window.as_float().to_bits(), + window.as_histogram().to_bits(), + "{window:?}" + ); + } + } + + // A synthetic zero at 2000ms with real samples at 2500ms and 4500ms, evaluated at 7000ms + // over a 6000ms range, gives the histogram route a sampled interval of 2500ms and the + // 2000ms average interval between the real samples. The end is 2500ms from the range end, + // past 1.1 * 2000ms, so it falls back to half the average interval; the start is 1500ms + // from the range start, inside that threshold, and the counter's zero point then takes it + // to 0: (2500 + 0 + 1000) / 2500 = 1.4. A float sample carries no start timestamp, so the + // float route samples the same 2000ms it measures its average over and answers + // (2000 + 0 + 1000) / 2000 = 1.5. The two fills disagree, which is what makes reading one + // anchor for the other observable rather than a change of arguments with no effect. + let two_anchor = Window { + sampled_from: 2_000, + average_from: 2_500, + last: 4_500, + samples_minus_one: 1, + eval_ts: 7_000, + range_ms: 6_000, + is_counter: true, + is_rate: false, + zero_point: Some(0.0), + }; + assert_eq!(two_anchor.as_histogram().to_bits(), 1.4f64.to_bits()); + assert_eq!(two_anchor.as_float().to_bits(), 1.5f64.to_bits()); + // Both directions an anchor can be replaced in are pinned, so neither the divisor nor the + // end's fallback can absorb the swap. Reading the average anchor for the sampled one + // shortens the sampled interval to 2000ms and answers 1.5; reading the sampled anchor for + // the average one widens the average interval to 2500ms, which puts the end back inside the + // threshold so it stays at 2500ms, and answers 2.0. The zero point takes the start to 0 + // whatever the average interval says, so the end has to fall past one threshold and inside + // the other for that: an end falling back under both average intervals, or under neither, + // answers the same either way and pins nothing. + assert_eq!( + Window { + sampled_from: two_anchor.average_from, + ..two_anchor + } + .as_histogram() + .to_bits(), + 1.5f64.to_bits() + ); + assert_eq!( + Window { + average_from: two_anchor.sampled_from, + ..two_anchor + } + .as_histogram() + .to_bits(), + 2.0f64.to_bits() + ); + } + + /// A synthetic zero puts the sampled anchor before the average anchor, and no float window + /// describes that: a float window starting at the zero would move its average interval there + /// too. The geometry is chosen so the two answers differ -- measuring the average from the + /// synthetic zero answers 1.75 where the real samples answer 1.25 -- so a swapped anchor cannot + /// pass by agreeing with itself. + #[test] + fn a_synthetic_zero_moves_only_the_sampled_anchor() { + // A zero at 1000ms, real samples at 2000ms and 3000ms, evaluated at 4500ms over a 4000ms + // range: the window opens at 500ms and the range ends 1500ms past the last sample. The + // sampled interval runs 2000ms from the zero, the average interval is the 1000ms between + // the real samples, so both ends lie past 1.1 * 1000ms and fall back to half of it, and the + // counter's zero point then takes the start to 0: (2000 + 0 + 500) / 2000 = 1.25. Measuring + // the average from the synthetic zero instead makes the average interval 2000ms, leaves + // both ends inside 1.1 * 2000ms and answers (2000 + 0 + 1500) / 2000 = 1.75. Both are + // pinned below, so swapping the anchors fails here instead of agreeing with itself. + let window = Window { + sampled_from: 1_000, + average_from: 2_000, + last: 3_000, + samples_minus_one: 1, + eval_ts: 4_500, + range_ms: 4_000, + is_counter: true, + is_rate: false, + zero_point: Some(0.0), + }; + assert_eq!(window.as_histogram().to_bits(), 1.25f64.to_bits()); + assert_eq!( + Window { + is_rate: true, + ..window + } + .as_histogram() + .to_bits(), + (1.25f64 / 4.0).to_bits() + ); + // The answer a swapped average anchor gives, pinned so that this geometry stays one the + // assertions above can tell apart. + assert_eq!( + Window { + average_from: 1_000, + ..window + } + .as_histogram() + .to_bits(), + 1.75f64.to_bits() + ); + // The float route on the same numbers starts its sampled interval at the first sample, + // which is a different window: 1.5 rather than 1.25. + assert_eq!(window.as_float().to_bits(), 1.5f64.to_bits()); + + // One real sample behind the zero leaves no average interval, so neither end is + // extrapolated and the factor is the identity. + let single = Window { + sampled_from: 1_000, + average_from: 2_000, + last: 2_000, + samples_minus_one: 0, + ..window + }; + assert_eq!(single.as_histogram().to_bits(), 1.0f64.to_bits()); + assert_eq!( + Window { + is_rate: true, + ..single + } + .as_histogram() + .to_bits(), + (1.0f64 / 4.0).to_bits() + ); + } +} diff --git a/src/promql/src/range_eval/histogram.rs b/src/promql/src/range_eval/histogram.rs index 1d4a2f5d2e0..3842226db82 100644 --- a/src/promql/src/range_eval/histogram.rs +++ b/src/promql/src/range_eval/histogram.rs @@ -26,7 +26,7 @@ use crate::grid_series::GridSample; use crate::histogram_eval::HistogramAggregate; use crate::histogram_eval::float_histogram::FloatHistogram; use crate::histogram_eval::sum_or_avg; -use crate::range_eval::function::{extrapolation_factor, seconds}; +use crate::range_eval::function::{ExtrapolationSpan, extrapolation_factor, seconds}; use crate::range_eval::{FloatWindow, RangeFunction, WindowKernel}; /// Histogram samples of one window at their shifted timestamps, in time order. @@ -241,52 +241,35 @@ pub(super) fn evaluate( return Ok(None); }; let effective_first_ts = synthetic_zero.unwrap_or(*first_ts); - let sample_count_minus_one = - histograms.len() - 1 + usize::from(synthetic_zero.is_some()); if *last_ts <= effective_first_ts { return Ok(None); } - let zero_point = (is_counter && synthetic_zero.is_none() && result.count > 0.0) - .then(|| first.count / result.count); - let factor = if let Some(synthetic_zero_start) = synthetic_zero { - // The sampled interval runs from the synthetic zero while the average interval is - // measured between the real samples, as `native_extrapolated_rate` does: they are - // two different spans, and taking one for the other shifts the average by a whole - // sample interval. - let sampled_interval = (*last_ts - synthetic_zero_start) as f64; - let average_interval = - (*last_ts - *first_ts) as f64 / (histograms.len() - 1) as f64; - let duration_to_start = 0.0; - let duration_to_end = (eval_ts - *last_ts) as f64; - let threshold = average_interval * 1.1; - let extrapolated = sampled_interval - + if duration_to_start < threshold { - duration_to_start - } else { - average_interval / 2.0 - } - + if duration_to_end < threshold { - duration_to_end - } else { - average_interval / 2.0 - }; - let factor = extrapolated / sampled_interval; - if function == Rate { - factor / (range_ms as f64 / 1000.0) - } else { - factor - } + // A synthetic zero *is* the counter's zero point, so nothing is extrapolated before it + // and `zero_point` says so with `0`; the counter instead extrapolates towards zero from + // its first real sample, and only when that sample is not itself negative. + let zero_point = if synthetic_zero.is_some() { + Some(0.0) } else { - extrapolation_factor( - (effective_first_ts, *last_ts), - sample_count_minus_one, - eval_ts, - range_ms, - is_counter, - function == Rate, - zero_point, - ) + (is_counter && result.count > 0.0 && first.count >= 0.0) + .then(|| first.count / result.count) }; + // The sampled interval runs from the synthetic zero while the average interval is + // measured between the real samples, as `native_extrapolated_rate` does: they are two + // different spans, and taking one for the other shifts the average by a whole sample + // interval. + let factor = extrapolation_factor( + ExtrapolationSpan { + sampled_from: effective_first_ts, + average_from: *first_ts, + last: *last_ts, + samples_minus_one: histograms.len() - 1, + }, + eval_ts, + range_ms, + is_counter, + function == Rate, + zero_point, + ); if synthetic_zero.is_some() && result.add(first).is_err() { record_warning( collector, diff --git a/src/promql/src/range_eval/tests.rs b/src/promql/src/range_eval/tests.rs index 6caba8f8b7b..de6d3fcf269 100644 --- a/src/promql/src/range_eval/tests.rs +++ b/src/promql/src/range_eval/tests.rs @@ -1318,20 +1318,21 @@ fn synthetic_zero_still_extrapolates_to_the_window_end() { } } - // Three real samples, where the two spans finally differ: the sampled interval runs from the - // synthetic zero (5000 - 1000) while the average interval is measured between the real samples - // (5000 - 2000 over two steps). Both are 4000 against 1500 here, so an average taken from the - // sampled interval instead divides by three and under-reports the increase. Prometheus answers - // 9 * 1.375 = 12.375. - for (function, expected) in [ - (RangeFunction::Increase, 12.375), - (RangeFunction::Rate, 2.0625), - ] { + // Two real samples, where the two spans finally differ: the sampled interval runs from the + // synthetic zero (3000 - 1000) while the average interval is measured between the real samples + // (3000 - 2000 over one step). That is 2000ms against 1000ms, so both ends fall past + // 1.1 * 1000ms and back to half of it, and the counter's zero point then takes the start to 0: + // the factor is (2000 + 0 + 500) / 2000 = 1.25. Measuring the average from the synthetic zero + // instead makes it 2000ms, leaves both ends inside 1.1 * 2000ms and answers 1.75 -- the swap + // this geometry fails on rather than passes by agreeing with itself. The zero adds its own + // sample back, so the count is (4 - 2) + 2 = 4, and Prometheus answers 4 * 1.25 = 5 for the + // increase and 4 * 1.25 / 4 = 1.25 for the rate. + for (function, expected) in [(RangeFunction::Increase, 5.0), (RangeFunction::Rate, 1.25)] { let mut evaluator = RangeEvaluator::try_new( &schema, - grid(6_500, 3_000, 1), + grid(4_500, 3_000, 1), RangeWindow { - range_ms: 6_000, + range_ms: 4_000, offset_ms: 0, anchor_ms: None, modifier: None, @@ -1343,19 +1344,18 @@ fn synthetic_zero_still_extrapolates_to_the_window_end() { &mut evaluator, &[ make_chunk(2_000, 2.0, Some(1_000), false), - make_chunk(3_500, 5.0, None, false), - make_chunk(5_000, 9.0, None, true), + make_chunk(3_000, 4.0, None, true), ], ); let series = fold_chunks(&chunks, evaluator.output_schema()).unwrap(); match series.fields.as_slice()[0].cursor().at(0).unwrap() { SampleRef::Histogram(histogram) => assert!( (histogram.count - expected).abs() < 1e-12, - "{function:?} over three samples: {} vs {expected}", + "{function:?} over two samples: {} vs {expected}", histogram.count ), other => panic!( - "expected histogram for {function:?} over three samples, got {}", + "expected histogram for {function:?} over two samples, got {}", describe_sample(Some(other)) ), } @@ -1397,6 +1397,90 @@ fn single_histogram_start_timestamp_is_compared_on_shifted_axis() { } } +/// A first histogram whose count is negative is not extrapolated towards zero. +/// +/// The zero point is `first.count / increase`, and a negative one would pull the sampled start +/// *backwards* (`duration_to_zero < duration_to_start`), handing the counter time it never spent +/// below zero. Prometheus guards it with `first.count >= 0`, which is what the histogram route now +/// reads too: without the guard this window answers 15 instead of 24. +#[test] +fn a_negative_first_histogram_count_is_not_extrapolated_towards_zero() { + let schema = raw_schema(1); + let make_chunk = |ts, count, is_last| { + let offsets = [0]; + let lengths = [2]; + // The buckets follow the count, so the increase is the count difference bucket by bucket. + let buckets = [count / 2.0; 2]; + let mut builder = HistogramBatchBuilder::new(); + let index = builder + .push(HistogramView { + schema: 0, + zero_threshold: 0.0, + zero_count: 0.0, + count, + sum: count, + reset_hint: CounterResetHint::Unknown, + start_timestamp: None, + custom_values: &[], + positive: BucketSideView { + span_offsets: &offsets, + span_lengths: &lengths, + buckets: &buckets, + }, + negative: BucketSideView { + span_offsets: &[], + span_lengths: &[], + buckets: &[], + }, + }) + .unwrap(); + let samples = Samples::try_from_histogram( + UInt32Array::from(vec![Some(index)]), + builder.finish().unwrap(), + ) + .unwrap(); + RawSeriesChunk::try_new( + &schema, + Buffer::from(0u64.to_le_bytes().to_vec()), + Some(metric(0)), + RawTimestamps::try_new(ScalarBuffer::from(vec![ts])).unwrap(), + SeriesFields::try_from_vec(vec![samples]).unwrap(), + is_last, + ) + .unwrap() + }; + let mut evaluator = RangeEvaluator::try_new( + &schema, + grid(6_500, 6_000, 1), + RangeWindow { + range_ms: 6_000, + offset_ms: 0, + anchor_ms: None, + modifier: None, + }, + RangeFunction::Increase, + ) + .unwrap(); + let chunks = collect_chunks( + &mut evaluator, + &[make_chunk(2_000, -3.0, false), make_chunk(5_000, 9.0, true)], + ); + let series = fold_chunks(&chunks, evaluator.output_schema()).unwrap(); + match series.fields.as_slice()[0].cursor().at(0).unwrap() { + // 9 - (-3) = 12 over the 3000ms between the samples, extrapolated 1500ms to the window's + // open end and 1500ms past the last sample: 12 * 2. + SampleRef::Histogram(histogram) => assert!( + (histogram.count - 24.0).abs() < 1e-12, + "increase {}", + histogram.count + ), + other => panic!( + "expected a histogram increase, got {}", + describe_sample(Some(other)) + ), + } +} + /// What a sample is, for a failure message: `SampleRef` carries a borrowed histogram payload and /// cannot be printed itself. fn describe_sample(sample: Option>) -> String {