mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 08:44:07 +00:00
Revert "perf(promql): stop building label-set tables where they cannot collide (#10412)"
This reverts commit 1fd9069418.
1626 lines
70 KiB
Rust
1626 lines
70 KiB
Rust
// Copyright 2023 Greptime Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! Range-vector window evaluation of raw series onto an evaluation grid.
|
|
|
|
use std::ops::Range;
|
|
use std::sync::Arc;
|
|
#[cfg(test)]
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
|
|
use common_query::prometheus::is_prometheus_stale_nan;
|
|
use common_query::promql_annotations::PromqlAnnotationCollector;
|
|
use datafusion::arrow::array::{Array, Float64Array};
|
|
use datafusion::arrow::buffer::Buffer;
|
|
use datafusion::error::{DataFusionError, Result};
|
|
use datatypes::series::SeriesLabels;
|
|
use datatypes::series::histogram::CounterResetHint;
|
|
use datatypes::series::raw::RawSeriesChunk;
|
|
use datatypes::series::samples::{
|
|
DenseSamplesRef, SampleRef, Samples, Series, SeriesFields, SeriesSchema,
|
|
};
|
|
use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis};
|
|
|
|
use crate::evaluated::EvaluatedChunk;
|
|
use crate::grid_series::{GridSample, GridSeriesBuilder, METADATA_LABELS, external};
|
|
use crate::histogram_eval::float_histogram::FloatHistogram;
|
|
|
|
mod function;
|
|
mod histogram;
|
|
#[cfg(test)]
|
|
mod tests;
|
|
|
|
pub use function::RangeFunction;
|
|
pub(crate) use function::quantile;
|
|
|
|
/// The samples a range selector reads at evaluation time `T`: source timestamps in
|
|
/// `(E - range_ms, E]`, where `E` is `T - offset_ms`, or `anchor_ms - offset_ms` with `@`. A
|
|
/// modifier widens that: `anchored` also reads the lookback before `E - range_ms`, `smoothed` the
|
|
/// lookback on both sides, and a scan feeding the window has to read that much more.
|
|
#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize, PartialEq, Hash, Eq)]
|
|
pub struct RangeWindow {
|
|
/// Positive window length in milliseconds.
|
|
pub range_ms: i64,
|
|
/// Selector offset in milliseconds; positive reads earlier samples.
|
|
pub offset_ms: i64,
|
|
/// The resolved `@` timestamp in milliseconds, with `start()` and `end()` already replaced
|
|
/// by the statement's start and end.
|
|
pub anchor_ms: Option<i64>,
|
|
/// The experimental `anchored` or `smoothed` selector modifier, which also reads samples
|
|
/// around the window.
|
|
pub modifier: Option<RangeModifier>,
|
|
}
|
|
|
|
/// The experimental range selector modifiers of Prometheus, which read samples beyond the
|
|
/// window to pin its boundary values. Not to be confused with an `@` anchor, which fixes the
|
|
/// window's end.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Hash)]
|
|
pub enum RangeModifier {
|
|
/// `anchored`: the window also reads the last sample at or before its start, up to the
|
|
/// statement's lookback before it, and takes it as the value at the start.
|
|
Anchored { lookback_ms: i64 },
|
|
/// `smoothed`: the window also reads samples up to the lookback before its start and after its
|
|
/// end, and interpolates the values at both boundaries.
|
|
Smoothed { lookback_ms: i64 },
|
|
}
|
|
|
|
impl RangeModifier {
|
|
/// How far before the window start the window also reads.
|
|
pub(crate) fn before_ms(self) -> i64 {
|
|
match self {
|
|
Self::Anchored { lookback_ms } | Self::Smoothed { lookback_ms } => lookback_ms,
|
|
}
|
|
}
|
|
|
|
/// How far after the window end the window also reads.
|
|
pub(crate) fn after_ms(self) -> i64 {
|
|
match self {
|
|
Self::Anchored { .. } => 0,
|
|
Self::Smoothed { lookback_ms } => lookback_ms,
|
|
}
|
|
}
|
|
|
|
pub(crate) fn name(self) -> &'static str {
|
|
match self {
|
|
Self::Anchored { .. } => "anchored",
|
|
Self::Smoothed { .. } => "smoothed",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The valid samples of one field inside one window, in source order.
|
|
pub(crate) struct FloatWindow<'a> {
|
|
/// Sample timestamps shifted by the offset onto the evaluation timeline; non-empty.
|
|
pub timestamps: &'a [i64],
|
|
pub values: &'a [f64],
|
|
/// The counter resets this window contains, ascending by sample. Empty for every function
|
|
/// that does not correct counters and for every window of a series that never reset.
|
|
pub counter_resets: &'a [CounterReset],
|
|
/// Evaluation timestamp `T`. `timestamps` lie in `(T - range_ms, T]`, except for an
|
|
/// `@`-anchored `predict_linear`, which evaluates the first point's window at every `T`, and
|
|
/// for a range modifier, which widens the window by the lookback as [`RangeWindow`] says.
|
|
pub eval_ts: i64,
|
|
pub range_ms: i64,
|
|
}
|
|
|
|
/// Computes one window result; `None` means the window has no result.
|
|
pub(crate) type WindowKernel = Box<dyn FnMut(FloatWindow<'_>) -> Option<f64> + Send>;
|
|
|
|
/// The windows of one field at consecutive grid points, as ranges into the field's retained
|
|
/// samples.
|
|
pub(crate) struct FloatWindows<'a> {
|
|
/// The retained samples, in time order.
|
|
pub timestamps: &'a [i64],
|
|
pub values: &'a [f64],
|
|
/// Counter resets over the retained samples, ascending by position.
|
|
pub resets: &'a [CounterReset],
|
|
/// Absolute position of `timestamps[0]`: a reset position is `base` plus a sample index.
|
|
pub base: u64,
|
|
/// One range per window, ascending. Each is a half-open index range `[start, end)`; the time
|
|
/// window it was derived from is left-open, `(T - range_ms, T]`.
|
|
pub windows: &'a [Range<usize>],
|
|
/// The evaluation timestamp of each window.
|
|
pub eval_ts: &'a [i64],
|
|
pub range_ms: i64,
|
|
}
|
|
|
|
impl FloatWindows<'_> {
|
|
/// The resets inside window `index`.
|
|
///
|
|
/// `windows` index the retained arrays as they stand in this batch, not absolutely: each
|
|
/// span is relative to `timestamps` and `values`, which is why `base` plus a span start is
|
|
/// the absolute position of the sample the span starts at.
|
|
pub(crate) fn resets_of(&self, index: usize) -> &[CounterReset] {
|
|
let span = &self.windows[index];
|
|
resets_between(
|
|
self.resets,
|
|
self.base + span.start as u64,
|
|
self.base + span.end as u64,
|
|
)
|
|
}
|
|
}
|
|
|
|
/// Computes a batch of one field's windows at once, appending one result per window to `out`.
|
|
pub(crate) type BatchKernel = Box<dyn FnMut(&FloatWindows<'_>, &mut Vec<Option<f64>>) + Send>;
|
|
|
|
/// Grid points accumulated before a batch kernel call. Sized to keep the per-field span and
|
|
/// column buffers in cache while staying large enough to amortise the call.
|
|
const EVALUATION_BATCH: usize = 512;
|
|
|
|
/// Evaluates range windows over raw series, one series at a time, handing out its results as they
|
|
/// become final.
|
|
///
|
|
/// Each series is pushed as its chunks in order. A push evaluates the grid points whose windows
|
|
/// the pushed samples complete, and hands them out as one [`EvaluatedChunk`], so a consumer can
|
|
/// stream a long series instead of waiting for its last chunk. One chunk may hold several runs of
|
|
/// positions, separated by stretches holding no value, and the axis names them. The series closes
|
|
/// on a chunk marked `is_last`, which is handed out even when it holds no position.
|
|
///
|
|
/// Fields are evaluated independently over their own valid samples; a row holding a stale
|
|
/// marker in any field is dropped, as the ordinary range selector does. Only the samples later
|
|
/// windows can still use are retained. Native histogram samples need a [`RangeFunction`]; an
|
|
/// evaluator built from a bare kernel rejects them.
|
|
pub struct RangeEvaluator {
|
|
grid: Arc<EvalGrid>,
|
|
/// The window with `offset_ms` replaced by the offset samples are shifted by.
|
|
window: RangeWindow,
|
|
replay: Replay,
|
|
kernel: WindowKernel,
|
|
function: Option<RangeFunction>,
|
|
annotations: Option<PromqlAnnotationCollector>,
|
|
output_schema: Arc<SeriesSchema>,
|
|
output: GridSeriesBuilder,
|
|
/// The grid of evaluated input series, such as the inner result of a subquery; `None` for
|
|
/// raw input.
|
|
input_grid: Option<GridSpec>,
|
|
series: Option<SeriesState>,
|
|
fields: Vec<FieldSamples>,
|
|
row: Vec<Option<GridSample>>,
|
|
drops_metric_name: bool,
|
|
/// Raised by the first non-empty window, where Prometheus validates function arguments.
|
|
window_error: Option<String>,
|
|
/// Chunks appended by the bulk path, so a test can prove that path is reached rather than
|
|
/// only that the output is right, which the per-sample path would also produce.
|
|
#[cfg(test)]
|
|
bulk_ingests: usize,
|
|
/// The batch form of `function`, when it has one and the input allows it. Present only for
|
|
/// `Replay::PerPoint`, whose windows advance with the grid; an anchored selector replays one
|
|
/// window and gains nothing from batching.
|
|
batch: Option<BatchKernel>,
|
|
/// Windows accumulated for the next batch call.
|
|
pending: PendingBatch,
|
|
/// Batch kernel calls made so far. Test-only: both dispatch shapes produce the same output,
|
|
/// so a test comparing only output could not tell which one ran.
|
|
#[cfg(test)]
|
|
batch_calls: usize,
|
|
}
|
|
|
|
/// Windows accumulated for the next batch call: one list per field.
|
|
struct PendingBatch {
|
|
/// Grid index of each accumulated point, ascending.
|
|
///
|
|
/// Not assumed contiguous: `evaluate` skips the grid points whose window ends before the
|
|
/// series has a sample, so a batch can span such a gap and must write each point to its own
|
|
/// row.
|
|
indices: Vec<usize>,
|
|
/// Evaluation timestamp of each accumulated point.
|
|
eval_ts: Vec<i64>,
|
|
/// The retained-sample range of each point's window, one list per field.
|
|
///
|
|
/// A span is relative to the retained arrays at the time it was recorded, not an absolute
|
|
/// index, so [`FieldSamples::advance`] invalidates it: that is why `evaluate` flushes the
|
|
/// batch before advancing.
|
|
spans: Vec<Vec<Range<usize>>>,
|
|
/// One output column per field, written by the batch kernel.
|
|
columns: Vec<Vec<Option<f64>>>,
|
|
/// Cursor into each field's retained timestamps; valid across batch flushes, but invalidated
|
|
/// when `FieldSamples::advance` drains the retained arrays.
|
|
ends: Vec<usize>,
|
|
/// Whether each cursor has been positioned by binary search since the last drain.
|
|
positioned: Vec<bool>,
|
|
}
|
|
|
|
impl PendingBatch {
|
|
fn with_fields(fields: usize) -> Self {
|
|
Self {
|
|
indices: Vec::new(),
|
|
eval_ts: Vec::new(),
|
|
spans: vec![Vec::new(); fields],
|
|
columns: vec![Vec::new(); fields],
|
|
ends: vec![0; fields],
|
|
positioned: vec![false; fields],
|
|
}
|
|
}
|
|
|
|
fn len(&self) -> usize {
|
|
self.eval_ts.len()
|
|
}
|
|
|
|
fn is_empty(&self) -> bool {
|
|
self.eval_ts.is_empty()
|
|
}
|
|
|
|
/// The bytes this batch's own vectors hold, by capacity.
|
|
fn bytes(&self) -> usize {
|
|
let spans: usize = self.spans.iter().map(|spans| spans.capacity()).sum();
|
|
let columns: usize = self.columns.iter().map(|column| column.capacity()).sum();
|
|
self.indices.capacity() * std::mem::size_of::<usize>()
|
|
+ self.eval_ts.capacity() * std::mem::size_of::<i64>()
|
|
+ spans * std::mem::size_of::<Range<usize>>()
|
|
+ columns * std::mem::size_of::<Option<f64>>()
|
|
+ self.ends.capacity() * std::mem::size_of::<usize>()
|
|
+ self.positioned.capacity() * std::mem::size_of::<bool>()
|
|
}
|
|
|
|
/// Drops the accumulated points, keeping the allocations.
|
|
fn clear_points(&mut self) {
|
|
self.indices.clear();
|
|
self.eval_ts.clear();
|
|
for spans in &mut self.spans {
|
|
spans.clear();
|
|
}
|
|
}
|
|
|
|
fn reset_cursors(&mut self) {
|
|
self.ends.fill(0);
|
|
self.positioned.fill(false);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
static POSITION_COMPARISONS: AtomicU64 = AtomicU64::new(0);
|
|
|
|
#[inline]
|
|
fn timestamp_le(timestamp: i64, eval_ts: i64) -> bool {
|
|
#[cfg(test)]
|
|
POSITION_COMPARISONS.fetch_add(1, Ordering::Relaxed);
|
|
timestamp <= eval_ts
|
|
}
|
|
|
|
/// Records one point for the next batch call: its evaluation timestamp and, per field, the range
|
|
/// of retained samples its window covers.
|
|
fn accumulate(pending: &mut PendingBatch, index: usize, eval_ts: i64, fields: &[FieldSamples]) {
|
|
debug_assert_eq!(pending.spans.len(), fields.len());
|
|
pending.indices.push(index);
|
|
pending.eval_ts.push(eval_ts);
|
|
for (field_index, (spans, field)) in pending.spans.iter_mut().zip(fields).enumerate() {
|
|
// Cursors refer to retained arrays, which are unchanged by flushes. `head` only moves
|
|
// forward; never revisit samples already behind every remaining window.
|
|
pending.ends[field_index] = pending.ends[field_index].max(field.head);
|
|
if !pending.positioned[field_index] {
|
|
pending.ends[field_index] = field.timestamps[field.head..]
|
|
.partition_point(|×tamp| timestamp_le(timestamp, eval_ts))
|
|
+ field.head;
|
|
pending.positioned[field_index] = true;
|
|
} else {
|
|
while pending.ends[field_index] < field.timestamps.len()
|
|
&& timestamp_le(field.timestamps[pending.ends[field_index]], eval_ts)
|
|
{
|
|
pending.ends[field_index] += 1;
|
|
}
|
|
}
|
|
spans.push(field.head..pending.ends[field_index]);
|
|
}
|
|
}
|
|
|
|
/// Runs the batch kernel over the accumulated points and appends its columns.
|
|
fn flush_batch(
|
|
kernel: &mut BatchKernel,
|
|
pending: &mut PendingBatch,
|
|
fields: &[FieldSamples],
|
|
range_ms: i64,
|
|
row: &mut Vec<Option<GridSample>>,
|
|
output: &mut GridSeriesBuilder,
|
|
#[cfg(test)] batch_calls: &mut usize,
|
|
) -> Result<()> {
|
|
if pending.is_empty() {
|
|
return Ok(());
|
|
}
|
|
// Counted here rather than at the call site, so the number a test reads is by construction
|
|
// the number of kernel calls that happened.
|
|
#[cfg(test)]
|
|
{
|
|
*batch_calls += 1;
|
|
}
|
|
for (index, field) in fields.iter().enumerate() {
|
|
let batch = FloatWindows {
|
|
timestamps: &field.timestamps,
|
|
values: &field.values,
|
|
resets: &field.resets,
|
|
base: field.base,
|
|
windows: &pending.spans[index],
|
|
eval_ts: &pending.eval_ts,
|
|
range_ms,
|
|
};
|
|
kernel(&batch, &mut pending.columns[index]);
|
|
}
|
|
for (offset, &grid_index) in pending.indices.iter().enumerate() {
|
|
row.clear();
|
|
for column in &pending.columns {
|
|
row.push(column[offset].map(GridSample::Float));
|
|
}
|
|
output.push_samples(grid_index, row)?;
|
|
}
|
|
pending.clear_points();
|
|
Ok(())
|
|
}
|
|
|
|
struct SeriesState {
|
|
/// `None` for evaluated input, which arrives as one complete series.
|
|
primary_key: Option<Buffer>,
|
|
metric: SeriesLabels,
|
|
/// Largest source timestamp retained so far, or `None` while the series holds no sample: a
|
|
/// closing chunk of a series that never produced a value carries no position to read.
|
|
last_ts: Option<i64>,
|
|
/// Next grid index to evaluate.
|
|
cursor: usize,
|
|
/// Whether the input, or the evaluated series, has ended. The state stays until the buffered
|
|
/// rows have been handed out: the closing chunk still needs its metric.
|
|
closing: bool,
|
|
/// Grid index after the last position handed out to the consumer.
|
|
emitted_end: usize,
|
|
}
|
|
|
|
impl SeriesState {
|
|
/// The bytes this series' identity holds, which the rows handed out do not cover.
|
|
fn identity_bytes(&self) -> usize {
|
|
// A scan slices the primary key out of a decoded buffer, so the bytes the key shows are not
|
|
// the bytes it keeps: a key of a few bytes pins the allocation it was sliced out of.
|
|
// Charging the slice length instead would show a handful of bytes for an allocation the
|
|
// size of a decoded batch. One evaluator holds one series at a time, and an evaluated input
|
|
// holds no key at all, so there is no second slice of the same allocation to dedupe.
|
|
let primary_key = self
|
|
.primary_key
|
|
.as_ref()
|
|
.map_or(0, |key| key.capacity().max(key.len()));
|
|
let metric: usize = self.metric.name().map_or(0, |name| name.len())
|
|
+ self
|
|
.metric
|
|
.tags()
|
|
.iter()
|
|
.map(|tag| tag.name.len() + tag.value.len())
|
|
.sum::<usize>();
|
|
std::mem::size_of::<SeriesState>() + primary_key + metric
|
|
}
|
|
}
|
|
|
|
/// How the windows of an `@`-anchored selector map to the grid.
|
|
///
|
|
/// Like Prometheus, an anchored selector is read once, as the window of the first grid point
|
|
/// with the offset rewritten to `start - (anchor - offset)`.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum Replay {
|
|
/// Not anchored: every grid point reads its own window.
|
|
PerPoint,
|
|
/// The first point's result is reported at every point (step-invariant functions).
|
|
Once,
|
|
/// The first point's window is evaluated at every point (`predict_linear`).
|
|
EveryPoint,
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct FieldSamples {
|
|
timestamps: Vec<i64>,
|
|
values: Vec<f64>,
|
|
/// Samples before `head` are behind every remaining window.
|
|
head: usize,
|
|
/// Native histogram samples, kept apart from the floats as Prometheus does.
|
|
histograms: Vec<(i64, FloatHistogram)>,
|
|
/// Histograms before `histogram_head` are behind every remaining window; their payloads are
|
|
/// released by [`advance_histograms`](Self::advance_histograms).
|
|
histogram_head: usize,
|
|
/// Estimated payload of the histograms this field still holds: the bucket lists and other
|
|
/// buffers the samples own, which no `size_of` of a slot shows.
|
|
///
|
|
/// Measured for a histogram when it is admitted and released with it, so a retained sample is
|
|
/// not walked again on every push. A histogram is never modified once admitted, so the payload
|
|
/// its release subtracts is the one its admission added. Samples already behind every window
|
|
/// keep their payload in the figure until the drain releases them.
|
|
histogram_bytes: usize,
|
|
/// Sequence number of `values[0]`. Absolute and monotonic across the series this evaluator
|
|
/// evaluates, so a reset keeps its position when `head` samples are dropped instead of every
|
|
/// index being renumbered. An empty field holds the number of samples already dropped, which
|
|
/// keeps positions increasing when the next series starts.
|
|
base: u64,
|
|
/// Resets between consecutive retained samples, ascending by position: exactly the ones a
|
|
/// window over these samples can still contain. Maintained only for the counter-correcting
|
|
/// functions, and released as the samples they reach back to are dropped.
|
|
resets: Vec<CounterReset>,
|
|
/// Whether `resets` is maintained for this field.
|
|
index_resets: bool,
|
|
}
|
|
|
|
/// One counter reset: a sample that dropped below the sample before it.
|
|
#[derive(Clone, Copy, Debug)]
|
|
pub(crate) struct CounterReset {
|
|
/// Position of the later sample, as an absolute sequence number. Its predecessor is
|
|
/// `position - 1`, so a window holding the samples `[start, end)` contains this reset
|
|
/// exactly when `start < position < end`.
|
|
pub position: u64,
|
|
/// The value the reset adds back: the sample at `position - 1`, which is inside the window
|
|
/// whenever the reset is.
|
|
pub previous_value: f64,
|
|
}
|
|
|
|
/// The resets inside the window over the absolute positions `[start, end)`.
|
|
///
|
|
/// Left-open: a reset at `start` would add back a sample behind the window.
|
|
fn resets_between(resets: &[CounterReset], start: u64, end: u64) -> &[CounterReset] {
|
|
let first = resets.partition_point(|reset| reset.position <= start);
|
|
let count = resets[first..].partition_point(|reset| reset.position < end);
|
|
&resets[first..first + count]
|
|
}
|
|
|
|
impl FieldSamples {
|
|
/// The bytes this field's buffers hold, by capacity: the samples, the history behind `head`
|
|
/// included, plus the estimated payload of the histograms still held.
|
|
fn bytes(&self) -> usize {
|
|
self.timestamps.capacity() * std::mem::size_of::<i64>()
|
|
+ self.values.capacity() * std::mem::size_of::<f64>()
|
|
+ self.histograms.capacity() * std::mem::size_of::<(i64, FloatHistogram)>()
|
|
+ self.resets.capacity() * std::mem::size_of::<CounterReset>()
|
|
+ self.histogram_bytes
|
|
}
|
|
|
|
fn new(index_resets: bool) -> Self {
|
|
Self {
|
|
index_resets,
|
|
..Self::default()
|
|
}
|
|
}
|
|
|
|
/// Appends one valid sample, recording a counter reset when it dropped below the sample
|
|
/// before it.
|
|
///
|
|
/// "Before it" means the previous entry of `values`, which leaves native histogram samples
|
|
/// out. A drop across an intervening histogram is therefore recorded as a reset of two
|
|
/// samples that are not neighbours. That is unobservable: a window holding both floats also
|
|
/// holds the histogram between them, so it is a mixed window, whose counter functions report
|
|
/// nothing; and a window starting after the earlier float excludes a reset on its first
|
|
/// sample. Should the mixed-window rules ever start reading the float resets, this has to
|
|
/// compare timestamps instead.
|
|
fn push(&mut self, timestamp: i64, value: f64) {
|
|
if self.index_resets
|
|
&& let Some(&previous_value) = self.values.last()
|
|
&& value < previous_value
|
|
{
|
|
self.resets.push(CounterReset {
|
|
position: self.base + self.values.len() as u64,
|
|
previous_value,
|
|
});
|
|
}
|
|
self.timestamps.push(timestamp);
|
|
self.values.push(value);
|
|
}
|
|
|
|
/// Appends samples that are all present, in bulk.
|
|
///
|
|
/// Same result as calling [`push`](Self::push) once per sample, including the reset scan,
|
|
/// which stays per-sample for a field that maintains the index; the two appends become two
|
|
/// `memcpy`s.
|
|
fn push_bulk(&mut self, timestamps: &[i64], values: &[f64]) {
|
|
debug_assert_eq!(timestamps.len(), values.len());
|
|
if self.index_resets {
|
|
let start = self.values.len();
|
|
let mut previous = self.values.last().copied();
|
|
for (offset, &value) in values.iter().enumerate() {
|
|
if let Some(previous_value) = previous
|
|
&& value < previous_value
|
|
{
|
|
self.resets.push(CounterReset {
|
|
position: self.base + (start + offset) as u64,
|
|
previous_value,
|
|
});
|
|
}
|
|
previous = Some(value);
|
|
}
|
|
}
|
|
self.timestamps.extend_from_slice(timestamps);
|
|
self.values.extend_from_slice(values);
|
|
}
|
|
|
|
/// Drops the samples before `head` and the resets whose predecessor is one of them.
|
|
fn advance(&mut self) {
|
|
let head = std::mem::take(&mut self.head);
|
|
if head == 0 {
|
|
return;
|
|
}
|
|
self.base += head as u64;
|
|
self.timestamps.drain(..head);
|
|
self.values.drain(..head);
|
|
// A reset at `base` would add back the last dropped sample, so it goes with it.
|
|
let dropped = self
|
|
.resets
|
|
.partition_point(|reset| reset.position <= self.base);
|
|
self.resets.drain(..dropped);
|
|
}
|
|
|
|
/// Drops the histograms before `histogram_head`, releasing the payload they held.
|
|
///
|
|
/// The dropped samples are measured as they go, which costs one measurement per dropped
|
|
/// histogram in total: the retained ones are never scanned again.
|
|
fn advance_histograms(&mut self) {
|
|
let head = std::mem::take(&mut self.histogram_head);
|
|
if head == 0 {
|
|
return;
|
|
}
|
|
let dropped: usize = self.histograms[..head]
|
|
.iter()
|
|
.map(|(_, histogram)| histogram.heap_bytes())
|
|
.sum();
|
|
self.histogram_bytes = self.histogram_bytes.saturating_sub(dropped);
|
|
self.histograms.drain(..head);
|
|
}
|
|
|
|
/// The resets inside the window holding `len` samples from `head`.
|
|
fn resets_in(&self, len: usize) -> &[CounterReset] {
|
|
let start = self.base + self.head as u64;
|
|
resets_between(&self.resets, start, start + len as u64)
|
|
}
|
|
}
|
|
|
|
/// The bytes the histograms a row holds box, beyond the slots they sit in.
|
|
///
|
|
/// A slot is an `Option<GridSample>`: a pointer and a tag. A result that is a native histogram
|
|
/// points at a [`FloatHistogram`] on the heap, which owns its bucket lists in turn, and both stay
|
|
/// alive between pushes. Only slots the last evaluated point left in place are counted: a push
|
|
/// clears the row.
|
|
fn boxed_bytes(row: &[Option<GridSample>]) -> usize {
|
|
row.iter()
|
|
.flatten()
|
|
.map(|sample| match sample {
|
|
GridSample::Histogram(histogram) => {
|
|
std::mem::size_of::<FloatHistogram>() + histogram.heap_bytes()
|
|
}
|
|
GridSample::Float(_) => 0,
|
|
})
|
|
.sum()
|
|
}
|
|
|
|
impl RangeEvaluator {
|
|
/// Creates an evaluator of `function` for the series of `input` onto `grid`.
|
|
pub fn try_new(
|
|
input: &SeriesSchema,
|
|
grid: Arc<EvalGrid>,
|
|
window: RangeWindow,
|
|
function: RangeFunction,
|
|
) -> Result<Self> {
|
|
let mut window_error = None;
|
|
if let RangeFunction::DoubleExponentialSmoothing { smoothing, trend } = function {
|
|
// Prometheus rejects factors outside (0, 1); NaN passes its check.
|
|
window_error = [("smoothing", smoothing), ("trend", trend)]
|
|
.into_iter()
|
|
.find(|(_, factor)| *factor <= 0.0 || *factor >= 1.0)
|
|
.map(|(name, factor)| {
|
|
format!("invalid {name} factor. Expected: 0 < factor < 1, got: {factor}")
|
|
});
|
|
}
|
|
if let Some(modifier) = window.modifier {
|
|
check_modifier(input, modifier, function)?;
|
|
}
|
|
let fold_offset = fold_offset(grid.spec(), window)?;
|
|
let step_invariant = !matches!(function, RangeFunction::PredictLinear(_));
|
|
if !step_invariant {
|
|
predict_linear_instants(grid.spec(), fold_offset)?;
|
|
}
|
|
// A modified window corrects counters over the samples it holds rather than through the
|
|
// reset index, and has no batch form.
|
|
let (kernel, index_resets) = match window.modifier {
|
|
Some(modifier) => (function.modified_kernel(modifier), false),
|
|
None => (
|
|
function.kernel(fold_offset),
|
|
function.indexes_counter_resets(),
|
|
),
|
|
};
|
|
let mut evaluator =
|
|
Self::with_kernel(input, grid, window, kernel, step_invariant, index_resets)?;
|
|
evaluator.drops_metric_name = function.drops_metric_name();
|
|
evaluator.window_error = window_error;
|
|
evaluator.function = Some(function);
|
|
// An anchored selector replays one window at every point, so its windows do not advance
|
|
// with the grid and a batch would only repeat the same call.
|
|
if matches!(evaluator.replay, Replay::PerPoint) && window.modifier.is_none() {
|
|
evaluator.batch = function.batch_kernel();
|
|
}
|
|
Ok(evaluator)
|
|
}
|
|
|
|
/// `step_invariant` tells whether an `@`-anchored result holds at every grid point.
|
|
/// `index_resets` tells whether the kernel corrects counters, which is the only case where
|
|
/// the per-field reset index is worth maintaining.
|
|
pub(crate) fn with_kernel(
|
|
input: &SeriesSchema,
|
|
grid: Arc<EvalGrid>,
|
|
mut window: RangeWindow,
|
|
kernel: WindowKernel,
|
|
step_invariant: bool,
|
|
index_resets: bool,
|
|
) -> Result<Self> {
|
|
if window.range_ms <= 0 {
|
|
return Err(DataFusionError::Plan(format!(
|
|
"range must be positive, got {} ms",
|
|
window.range_ms
|
|
)));
|
|
}
|
|
let replay = match (window.anchor_ms, step_invariant) {
|
|
(None, _) => Replay::PerPoint,
|
|
(Some(_), true) => Replay::Once,
|
|
(Some(_), false) => Replay::EveryPoint,
|
|
};
|
|
window.offset_ms = fold_offset(grid.spec(), window)?;
|
|
let output_schema = Arc::new(SeriesSchema {
|
|
fields: input.fields.clone(),
|
|
eval_grid: Some(grid.clone()),
|
|
});
|
|
Ok(Self {
|
|
grid,
|
|
window,
|
|
replay,
|
|
kernel,
|
|
function: None,
|
|
annotations: None,
|
|
output: GridSeriesBuilder::try_new(output_schema.clone())?,
|
|
input_grid: input.eval_grid.as_ref().map(|grid| grid.spec()),
|
|
output_schema,
|
|
series: None,
|
|
fields: input
|
|
.fields
|
|
.iter()
|
|
.map(|_| FieldSamples::new(index_resets))
|
|
.collect(),
|
|
row: Vec::with_capacity(input.fields.len()),
|
|
drops_metric_name: false,
|
|
window_error: None,
|
|
#[cfg(test)]
|
|
bulk_ingests: 0,
|
|
batch: None,
|
|
pending: PendingBatch::with_fields(input.fields.len()),
|
|
#[cfg(test)]
|
|
batch_calls: 0,
|
|
})
|
|
}
|
|
|
|
/// Chunks appended without the per-sample loop. Test-only: both paths produce the same
|
|
/// output by construction, so a test that only compared output could not tell whether the
|
|
/// bulk path ran at all.
|
|
#[cfg(test)]
|
|
pub(crate) fn bulk_ingest_count(&self) -> usize {
|
|
self.bulk_ingests
|
|
}
|
|
|
|
/// Batch kernel calls made so far. Test-only: both dispatch shapes produce the same output,
|
|
/// so a test comparing only output could not tell which one ran.
|
|
#[cfg(test)]
|
|
pub(crate) fn batch_call_count(&self) -> usize {
|
|
self.batch_calls
|
|
}
|
|
|
|
/// Sets the collector used for annotations produced by histogram range evaluation.
|
|
pub fn set_annotations(&mut self, collector: Option<PromqlAnnotationCollector>) {
|
|
self.annotations = collector;
|
|
}
|
|
|
|
/// An estimate of the bytes this evaluator keeps between pushes.
|
|
///
|
|
/// Counted by capacity, so it is what the storage holds rather than what it uses: the retained
|
|
/// samples per field (timestamps, values, the reset index and the native histograms with the
|
|
/// payload each of them owns), the row a push fills with the histograms it boxes, the identity
|
|
/// of the open series and the pending batch's own vectors. Left out are the kernel's scratch
|
|
/// buffers, this struct's `Vec` headers and the grid, which every chunk addressed on it shares.
|
|
///
|
|
/// The figure is an estimate, not a ledger of every allocation: a histogram's payload is
|
|
/// measured once when it is admitted and released with it, so neither the retained samples nor
|
|
/// the transient allocations of a push are walked again.
|
|
///
|
|
/// A push is the only caller that also holds the incoming chunk, and it charges that itself for
|
|
/// as long as the call runs. The figure must not be per-chunk either: charging every chunk
|
|
/// consumed would make the charge grow with the chunk count, so a fixed window over a long
|
|
/// series would read as an allocation failure even though the window holds no more samples.
|
|
pub(crate) fn retained_bytes(&self) -> usize {
|
|
let fields: usize = self.fields.iter().map(FieldSamples::bytes).sum();
|
|
let row = self.row.capacity() * std::mem::size_of::<Option<GridSample>>();
|
|
fields
|
|
+ row
|
|
+ boxed_bytes(&self.row)
|
|
+ self.series.as_ref().map_or(0, SeriesState::identity_bytes)
|
|
+ self.pending.bytes()
|
|
}
|
|
|
|
/// An upper bound of the rows the next push hands out, for input whose last sample is at
|
|
/// `last_ts` (`None` when it holds none) and which closes its series when `last`.
|
|
///
|
|
/// A push writes at most one row per point it decides, from the series' cursor -- the first
|
|
/// point for a series not started yet. Unless the series closes, it decides no point past the
|
|
/// time the input has reached; and a point holds a row only while its window still reaches the
|
|
/// latest sample, so the points after that one's window are decided empty. A replayed `@`
|
|
/// result fills every grid point from the one window it evaluates first.
|
|
pub(crate) fn output_rows_bound(&self, last_ts: Option<i64>, last: bool) -> usize {
|
|
let spec = self.grid.spec();
|
|
let len = usize::try_from(spec.len).unwrap_or(usize::MAX);
|
|
let state = self.series.as_ref();
|
|
let Some(latest) = last_ts.or_else(|| state.and_then(|state| state.last_ts)) else {
|
|
return 0;
|
|
};
|
|
let cursor = state.map_or(0, |state| state.cursor);
|
|
if !matches!(self.replay, Replay::PerPoint) {
|
|
return if cursor == 0 { len } else { 0 };
|
|
}
|
|
// The grid points whose evaluation time is before `bound`.
|
|
let (start, step) = (i128::from(spec.start_ms), i128::from(spec.step_ms));
|
|
let points_before = |bound: i128| {
|
|
let offset = bound - start;
|
|
if offset <= 0 {
|
|
return 0;
|
|
}
|
|
usize::try_from((offset + step - 1) / step).map_or(len, |points| points.min(len))
|
|
};
|
|
let shifted = i128::from(latest) + i128::from(self.window.offset_ms);
|
|
let before = self
|
|
.window
|
|
.modifier
|
|
.map_or(0, |modifier| i128::from(modifier.before_ms()));
|
|
let mut end = points_before(shifted + i128::from(self.window.range_ms) + before);
|
|
if !last && last_ts.is_some() {
|
|
end = end.min(points_before(shifted + 1));
|
|
}
|
|
end.saturating_sub(cursor)
|
|
}
|
|
|
|
/// Whether this evaluator can answer with a native histogram.
|
|
pub(crate) fn yields_histograms(&self) -> bool {
|
|
self.function.is_some_and(RangeFunction::yields_histograms)
|
|
}
|
|
|
|
/// The heap bytes of the widest native histogram this evaluator retains, `None` when it
|
|
/// retains none.
|
|
pub(crate) fn widest_retained_histogram(&self) -> Option<usize> {
|
|
self.fields
|
|
.iter()
|
|
.flat_map(|field| &field.histograms)
|
|
.map(|(_, histogram)| histogram.heap_bytes())
|
|
.max()
|
|
}
|
|
|
|
/// Schema of the evaluated series: the input fields on the evaluation grid.
|
|
pub fn output_schema(&self) -> &Arc<SeriesSchema> {
|
|
&self.output_schema
|
|
}
|
|
|
|
/// Consumes the next chunk of the current series, or starts a new series.
|
|
///
|
|
/// Returns every position this chunk decided, in one chunk addressed absolutely on the
|
|
/// evaluation grid; `None` when it decided no position holding a value. A decided position
|
|
/// without a result is a row of no chunk, so the positions between two runs of one chunk hold
|
|
/// no value and cost nothing. One push hands out at most one chunk.
|
|
///
|
|
/// A series always closes exactly once: on the chunk holding its last rows, or on an empty
|
|
/// chunk after it when nothing is left. The evaluator must not be reused after an error.
|
|
pub fn push(&mut self, chunk: &RawSeriesChunk) -> Result<Option<EvaluatedChunk>> {
|
|
if self.input_grid.is_some() {
|
|
return Err(DataFusionError::Execution(
|
|
"raw series chunk pushed to an evaluator of evaluated series".into(),
|
|
));
|
|
}
|
|
self.check_chunk(chunk)?;
|
|
self.ingest(chunk.time.as_buffer(), &chunk.fields, true)?;
|
|
let last_ts = chunk.time.as_buffer()[chunk.time.len() - 1];
|
|
if !chunk.is_last {
|
|
// Later chunks only hold samples after `last_ts`.
|
|
self.evaluate(Some(
|
|
i128::from(last_ts) + i128::from(self.window.offset_ms),
|
|
))?;
|
|
return self.emit();
|
|
}
|
|
self.close_series()
|
|
}
|
|
|
|
/// Evaluates one run of an evaluated series, such as the inner result of a subquery: the
|
|
/// value at each grid point is a sample at that time. Stale markers are kept as values,
|
|
/// since the input is computed.
|
|
///
|
|
/// `last` says whether this delivery closes its series. A stream hands over the same logical
|
|
/// series more than once -- a prefix now, its closing chunk later -- and the windows of a later
|
|
/// delivery read the samples of the earlier ones, so the state a series retains lives on across
|
|
/// them: only the `last` call ends it. Handing the deliveries over without saying which is last
|
|
/// would close the series early and lose exactly the samples a window across the boundary
|
|
/// needs.
|
|
///
|
|
/// One delivery may hold several runs of positions, separated by stretches holding no value;
|
|
/// they all reach the caller in one chunk, addressed by the axis. A delivery that holds no
|
|
/// position at all is a legal close when it is `last`, and an error otherwise.
|
|
pub fn push_series(&mut self, series: &Series, last: bool) -> Result<Option<EvaluatedChunk>> {
|
|
let time = &series.time;
|
|
let Some(grid) = time.grid() else {
|
|
return Err(DataFusionError::Execution(
|
|
"evaluated range input must be a grid series".into(),
|
|
));
|
|
};
|
|
if self.input_grid != Some(grid.spec())
|
|
|| series.fields.len() != self.fields.len()
|
|
|| series.fields.iter().any(|field| field.len() != time.len())
|
|
{
|
|
return Err(DataFusionError::Execution(
|
|
"evaluated series does not match the input schema".into(),
|
|
));
|
|
}
|
|
// The input may skip positions: it is ingested by the positions it carries, and the points
|
|
// it skips hold no sample, which is exactly what a hole means.
|
|
let timestamps = time.timestamps().map_err(external)?;
|
|
let last_ts = timestamps.last().copied();
|
|
if !last && last_ts.is_none() {
|
|
// An empty delivery inside a series decides nothing: it is indistinguishable from a
|
|
// stall, so only the closing `is_last` one is a legal close.
|
|
return Err(DataFusionError::Execution(
|
|
"evaluated range input holds no position before its last run".into(),
|
|
));
|
|
}
|
|
match &mut self.series {
|
|
None => {
|
|
self.output.begin_series();
|
|
self.series = Some(SeriesState {
|
|
primary_key: None,
|
|
metric: series.metric.clone(),
|
|
last_ts,
|
|
cursor: 0,
|
|
closing: false,
|
|
emitted_end: 0,
|
|
});
|
|
}
|
|
Some(state) => {
|
|
// A run continues its series; the earlier runs' samples stay retained for it.
|
|
if state.primary_key.is_some() {
|
|
return Err(DataFusionError::Execution(
|
|
"evaluated series pushed inside a raw series".into(),
|
|
));
|
|
}
|
|
if state.closing {
|
|
return Err(DataFusionError::Execution(
|
|
"evaluated series pushed after it closed".into(),
|
|
));
|
|
}
|
|
if state.metric != series.metric {
|
|
return Err(DataFusionError::Execution(
|
|
"evaluated series changed identity before its last run".into(),
|
|
));
|
|
}
|
|
if let Some(last_ts) = last_ts {
|
|
if let Some(previous) = state.last_ts
|
|
&& timestamps[0] <= previous
|
|
{
|
|
return Err(DataFusionError::Execution(format!(
|
|
"evaluated series runs are not increasing: {} after {previous}",
|
|
timestamps[0]
|
|
)));
|
|
}
|
|
state.last_ts = Some(last_ts);
|
|
}
|
|
}
|
|
}
|
|
if let Some(last_ts) = last_ts {
|
|
self.ingest(×tamps, &series.fields, false)?;
|
|
if last {
|
|
return self.close_series();
|
|
}
|
|
// Later runs only hold samples after `last_ts`, so every point up to it is decided.
|
|
// The samples are shifted onto the evaluation timeline by the selector offset, so the
|
|
// ready point is on that timeline too.
|
|
self.evaluate(Some(
|
|
i128::from(last_ts) + i128::from(self.window.offset_ms),
|
|
))?;
|
|
return self.emit();
|
|
}
|
|
// A closing chunk holding no position: there is nothing to ingest, but the samples the
|
|
// earlier deliveries left retained still decide the windows that follow them, so the close
|
|
// finishes them rather than ending the series on the spot.
|
|
debug_assert!(last, "an empty chunk that is not last is rejected above");
|
|
self.close_series()
|
|
}
|
|
|
|
/// Evaluates the rest of the series of the last input chunk and closes it.
|
|
///
|
|
/// The rows still buffered are handed out here as one chunk, and the chunk that sees nothing
|
|
/// behind it carries `is_last`.
|
|
fn close_series(&mut self) -> Result<Option<EvaluatedChunk>> {
|
|
self.evaluate(None)?;
|
|
self.series
|
|
.as_mut()
|
|
.expect("every path that closes a series started one")
|
|
.closing = true;
|
|
self.emit()
|
|
}
|
|
|
|
/// Hands out the rows buffered so far as one chunk, and closes the series when nothing is left.
|
|
///
|
|
/// The chunk holds every row decided since the last hand-out, in one go and in one [`Series`]:
|
|
/// several runs of positions may sit inside it, separated by stretches holding no value, and
|
|
/// the axis names them. A position decided to hold none costs no row, so it lies between the
|
|
/// runs rather than inside one. `is_last` closes the series, and a series always closes exactly
|
|
/// once: on the chunk that carries its last rows, or on an empty chunk after it when no row is
|
|
/// left.
|
|
fn emit(&mut self) -> Result<Option<EvaluatedChunk>> {
|
|
let Some(state) = self.series.as_ref() else {
|
|
return Ok(None);
|
|
};
|
|
let closing = state.closing;
|
|
let emitted_end = state.emitted_end;
|
|
let mut metric = state.metric.clone();
|
|
if self.drops_metric_name {
|
|
metric.remove_all(&METADATA_LABELS);
|
|
}
|
|
let Some(series) = self.output.take_buffered(metric)? else {
|
|
return if closing {
|
|
self.empty_chunk(emitted_end)
|
|
} else {
|
|
Ok(None)
|
|
};
|
|
};
|
|
// The builder addresses every position it hands out, so the chunk's bounds come from the
|
|
// axis itself: `first_position` is where it starts, and `last_position` where it ends.
|
|
let start = series
|
|
.time
|
|
.first_position()
|
|
.expect("the builder addresses every row on the grid");
|
|
let end = series.time.last_position().map_or(start, |last| last + 1);
|
|
debug_assert!(start >= emitted_end, "a chunk never goes back");
|
|
if closing {
|
|
// The buffer was drained into this chunk, so nothing of the series is left behind it.
|
|
self.series = None;
|
|
EvaluatedChunk::try_new(series, true).map(Some)
|
|
} else {
|
|
self.series
|
|
.as_mut()
|
|
.expect("checked at the entry of emit")
|
|
.emitted_end = end;
|
|
EvaluatedChunk::try_new(series, false).map(Some)
|
|
}
|
|
}
|
|
|
|
/// A closing chunk without values, addressed after the last position handed out.
|
|
fn empty_chunk(&mut self, at: usize) -> Result<Option<EvaluatedChunk>> {
|
|
let state = self.series.as_ref().expect("checked by the caller");
|
|
let mut metric = state.metric.clone();
|
|
if self.drops_metric_name {
|
|
metric.remove_all(&METADATA_LABELS);
|
|
}
|
|
let empty = Samples::from_float(Float64Array::from(Vec::<f64>::new()));
|
|
let fields = SeriesFields::try_from_vec(vec![empty; self.output_schema.fields.len()])
|
|
.map_err(external)?;
|
|
let time = GridRange::try_new(self.grid.clone(), at, 0).map_err(external)?;
|
|
let series = Series::try_new(&self.output_schema, metric, TimeAxis::Grid(time), fields)
|
|
.map_err(external)?;
|
|
self.series = None;
|
|
EvaluatedChunk::try_new(series, true).map(Some)
|
|
}
|
|
|
|
/// Checks that the input did not end inside a series, with rows nobody took.
|
|
pub fn finish(self) -> Result<()> {
|
|
if self.series.is_some() {
|
|
return Err(DataFusionError::Execution(
|
|
"raw series input ended before the last chunk of a series".into(),
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn check_chunk(&mut self, chunk: &RawSeriesChunk) -> Result<()> {
|
|
let primary_key = Some(chunk.primary_key.as_slice());
|
|
let time = chunk.time.as_buffer();
|
|
if time.is_empty()
|
|
|| chunk.fields.len() != self.fields.len()
|
|
|| chunk.fields.iter().any(|field| field.len() != time.len())
|
|
{
|
|
return Err(DataFusionError::Execution(format!(
|
|
"raw series chunk with {} timestamps and {} fields does not match {} schema fields",
|
|
time.len(),
|
|
chunk.fields.len(),
|
|
self.fields.len()
|
|
)));
|
|
}
|
|
match &mut self.series {
|
|
None => {
|
|
let metric = chunk.metric.clone().ok_or_else(|| {
|
|
DataFusionError::Execution(
|
|
"range evaluation requires materialized series labels".into(),
|
|
)
|
|
})?;
|
|
self.output.begin_series();
|
|
self.series = Some(SeriesState {
|
|
primary_key: Some(chunk.primary_key.clone()),
|
|
metric,
|
|
last_ts: Some(time[time.len() - 1]),
|
|
cursor: 0,
|
|
closing: false,
|
|
emitted_end: 0,
|
|
});
|
|
}
|
|
Some(state) => {
|
|
if state.primary_key.as_ref().map(|key| key.as_slice()) != primary_key
|
|
|| chunk.metric.as_ref() != Some(&state.metric)
|
|
{
|
|
return Err(DataFusionError::Execution(
|
|
"raw series identity changed before its last chunk".into(),
|
|
));
|
|
}
|
|
if let Some(last_ts) = state.last_ts
|
|
&& time[0] <= last_ts
|
|
{
|
|
return Err(DataFusionError::Execution(format!(
|
|
"raw series timestamps are not increasing across chunks: {} after {last_ts}",
|
|
time[0]
|
|
)));
|
|
}
|
|
state.last_ts = Some(time[time.len() - 1]);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Appends the chunk in bulk when every field is an entirely present, unshifted float column.
|
|
///
|
|
/// Returns `false` when any condition of the per-sample path applies, so the caller can run
|
|
/// it. A chunk carrying a stale marker takes that path too: the marker drops the whole row,
|
|
/// and reproducing that here would cost the same scan the loop already makes.
|
|
///
|
|
/// A field mixing floats and native histograms never qualifies: `dense()` reports
|
|
/// [`DenseSamplesRef::Mixed`] for it, keeping its windows on the histogram path where the
|
|
/// mixed-window rules decide the result.
|
|
fn ingest_in_bulk(
|
|
&mut self,
|
|
timestamps: &[i64],
|
|
fields: &SeriesFields,
|
|
drop_stale: bool,
|
|
) -> bool {
|
|
// Lengths are checked by `check_chunk` for raw input and by `push_series` for evaluated
|
|
// input, so only the shape of the columns is decided here.
|
|
if self.window.offset_ms != 0 {
|
|
return false;
|
|
}
|
|
let mut columns = Vec::with_capacity(fields.len());
|
|
for field in fields.iter() {
|
|
let Some(DenseSamplesRef::Float(values)) = field.dense() else {
|
|
return false;
|
|
};
|
|
if values.null_count() != 0 {
|
|
return false;
|
|
}
|
|
columns.push(values);
|
|
}
|
|
// A kept sample's shifted time must fit `i64`; with no offset it is the source timestamp,
|
|
// which `RawTimestamps` already validated as increasing and which the frame widened into
|
|
// this array without narrowing, so there is nothing left to check here.
|
|
if drop_stale
|
|
&& columns.iter().any(|values| {
|
|
values
|
|
.values()
|
|
.iter()
|
|
.any(|value| is_prometheus_stale_nan(*value))
|
|
})
|
|
{
|
|
return false;
|
|
}
|
|
for (field, values) in self.fields.iter_mut().zip(columns) {
|
|
field.push_bulk(timestamps, values.values());
|
|
}
|
|
#[cfg(test)]
|
|
{
|
|
self.bulk_ingests += 1;
|
|
}
|
|
true
|
|
}
|
|
|
|
/// Retains the samples of `fields` at `timestamps`. From `raw` input a row holding a stale
|
|
/// marker in any field is dropped; evaluated input keeps it as a value.
|
|
fn ingest(&mut self, timestamps: &[i64], fields: &SeriesFields, raw: bool) -> Result<()> {
|
|
if self.ingest_in_bulk(timestamps, fields, raw) {
|
|
return Ok(());
|
|
}
|
|
let mut samples: Vec<_> = fields.iter().map(|field| field.iter()).collect();
|
|
let mut row = Vec::with_capacity(samples.len());
|
|
for ×tamp in timestamps {
|
|
row.clear();
|
|
let mut stale = false;
|
|
for field in &mut samples {
|
|
let sample = field.next().expect("field length checked by the caller");
|
|
match &sample {
|
|
SampleRef::Missing => {}
|
|
SampleRef::Float(value) => stale |= is_prometheus_stale_nan(*value),
|
|
SampleRef::Histogram(view) => {
|
|
if self.function.is_none() {
|
|
return Err(DataFusionError::NotImplemented(
|
|
"range evaluation of histogram samples".into(),
|
|
));
|
|
}
|
|
stale |= is_prometheus_stale_nan(view.sum);
|
|
}
|
|
}
|
|
row.push(sample);
|
|
}
|
|
if stale && raw {
|
|
continue;
|
|
}
|
|
// Like RangeManipulate, any kept sample whose shifted time leaves i64 fails the series.
|
|
let shifted = timestamp
|
|
.checked_add(self.window.offset_ms)
|
|
.ok_or_else(|| {
|
|
DataFusionError::Execution(format!(
|
|
"sample timestamp {timestamp} overflows with offset {} ms",
|
|
self.window.offset_ms
|
|
))
|
|
})?;
|
|
// Refused here rather than when a point is evaluated: a smoothed point can be decided
|
|
// before the rest of its window arrives, so whether a later histogram is seen would
|
|
// depend on how the input is chunked.
|
|
if let Some(modifier) = self.window.modifier
|
|
&& row
|
|
.iter()
|
|
.any(|sample| matches!(sample, SampleRef::Histogram(_)))
|
|
&& self.decided_window_reads(shifted)
|
|
{
|
|
return Err(DataFusionError::NotImplemented(format!(
|
|
"the {} range modifier over native histogram samples",
|
|
modifier.name()
|
|
)));
|
|
}
|
|
for (field, sample) in self.fields.iter_mut().zip(&row) {
|
|
match sample {
|
|
SampleRef::Missing => {}
|
|
SampleRef::Float(value) => {
|
|
field.push(shifted, *value);
|
|
}
|
|
SampleRef::Histogram(view) => {
|
|
let mut h = FloatHistogram::try_from_view(view)?;
|
|
// Like Prometheus' `evalSubquery`: an explicit counter reset hint
|
|
// describes the sample's raw neighbours, not its evaluated ones.
|
|
if !raw && h.reset_hint != CounterResetHint::Gauge {
|
|
h.reset_hint = CounterResetHint::Unknown;
|
|
}
|
|
field.histogram_bytes =
|
|
field.histogram_bytes.saturating_add(h.heap_bytes());
|
|
field.histograms.push((shifted, h));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Whether a sample at `timestamp` on the evaluation timeline lies in the window of a point
|
|
/// this evaluator decides: a window ending at `E` reads `(E - range - before, E + after]`, and
|
|
/// only the first point's window is read when the result is replayed.
|
|
fn decided_window_reads(&self, timestamp: i64) -> bool {
|
|
let (before, after) = self.window.modifier.map_or((0, 0), |modifier| {
|
|
(
|
|
i128::from(modifier.before_ms()),
|
|
i128::from(modifier.after_ms()),
|
|
)
|
|
});
|
|
let spec = self.grid.spec();
|
|
let ends = match self.replay {
|
|
Replay::PerPoint => i128::from(spec.len),
|
|
Replay::Once | Replay::EveryPoint => 1,
|
|
};
|
|
let (start, step) = (i128::from(spec.start_ms), i128::from(spec.step_ms));
|
|
// The window ends that read the sample are those in `[timestamp - after, timestamp +
|
|
// range + before)`; the first grid point at or after the lower bound decides.
|
|
let lowest = i128::from(timestamp) - after;
|
|
let highest = i128::from(timestamp) + i128::from(self.window.range_ms) + before;
|
|
let first = if lowest <= start {
|
|
0
|
|
} else {
|
|
(lowest - start + step - 1) / step
|
|
};
|
|
first < ends && start + first * step < highest
|
|
}
|
|
|
|
/// Evaluates grid points up to `ready` on the evaluation timeline, or all remaining points
|
|
/// when `ready` is `None`.
|
|
fn evaluate(&mut self, ready: Option<i128>) -> Result<()> {
|
|
let points_read = self.points_read();
|
|
let state = self.series.as_mut().expect("checked by check_chunk");
|
|
let spec = self.grid.spec();
|
|
let grid = self.grid.timestamps().map_err(external)?;
|
|
// A modifier also reads samples up to the lookback ahead of the window start, so they are
|
|
// kept that much longer.
|
|
let before = self
|
|
.window
|
|
.modifier
|
|
.map_or(0, |modifier| i128::from(modifier.before_ms()));
|
|
// `smoothed` interpolates the window end with the first sample at or after it, which can
|
|
// arrive up to the lookback later. A point is decided once every field holds a sample at
|
|
// or after it, or once the lookback past it has arrived. The input's last row does not say
|
|
// that alone: a stale row is dropped, and a field can hold no value in it.
|
|
let ready = match (ready, self.window.modifier) {
|
|
(Some(ready), Some(RangeModifier::Smoothed { lookback_ms })) => {
|
|
let held = self
|
|
.fields
|
|
.iter()
|
|
.map(|field| {
|
|
field
|
|
.timestamps
|
|
.last()
|
|
.map_or(i128::MIN, |&t| i128::from(t))
|
|
})
|
|
.min()
|
|
.unwrap_or(i128::MIN);
|
|
Some(held.max(ready - i128::from(lookback_ms)))
|
|
}
|
|
(ready, _) => ready,
|
|
};
|
|
let range = i128::from(self.window.range_ms) + before;
|
|
let mut empty_tail = false;
|
|
while state.cursor < points_read {
|
|
let eval_ts = grid[state.cursor];
|
|
let lower = i128::from(eval_ts) - range;
|
|
let mut earliest = None;
|
|
// Later windows start no earlier, so samples behind this one go even when the point
|
|
// itself is not ready yet.
|
|
for field in &mut self.fields {
|
|
field.head += field.timestamps[field.head..]
|
|
.partition_point(|×tamp| i128::from(timestamp) <= lower);
|
|
field.histogram_head += field.histograms[field.histogram_head..]
|
|
.partition_point(|(timestamp, _)| i128::from(*timestamp) <= lower);
|
|
let first_histogram = field.histograms.get(field.histogram_head);
|
|
for ×tamp in field
|
|
.timestamps
|
|
.get(field.head)
|
|
.into_iter()
|
|
.chain(first_histogram.map(|(timestamp, _)| timestamp))
|
|
{
|
|
earliest = Some(earliest.map_or(timestamp, |min: i64| min.min(timestamp)));
|
|
}
|
|
}
|
|
if ready.is_some_and(|ready| i128::from(eval_ts) > ready) {
|
|
break;
|
|
}
|
|
// No retained sample: every point from here on is decided too. A sample that still
|
|
// has to arrive lies beyond the window of every one of them, so none can gain a value.
|
|
let Some(earliest) = earliest else {
|
|
empty_tail = true;
|
|
break;
|
|
};
|
|
if earliest > eval_ts {
|
|
// Skip the points whose windows end before the earliest sample. A smoothed window
|
|
// reads past its end, but without a sample at or before its end it has no result,
|
|
// as Prometheus returns nothing when the first sample is after the end.
|
|
let step = i128::from(spec.step_ms);
|
|
let steps = (i128::from(earliest) - i128::from(spec.start_ms) + step - 1) / step;
|
|
state.cursor = usize::try_from(steps).map_or(grid.len(), |i| i.min(grid.len()));
|
|
continue;
|
|
}
|
|
if let Some(error) = &self.window_error {
|
|
return Err(DataFusionError::Execution(error.clone()));
|
|
}
|
|
|
|
let mut evaluator = RowEvaluator {
|
|
fields: &self.fields,
|
|
kernel: &mut self.kernel,
|
|
function: self.function,
|
|
annotations: self.annotations.as_ref(),
|
|
range_ms: self.window.range_ms,
|
|
offset_ms: self.window.offset_ms,
|
|
modifier: self.window.modifier,
|
|
};
|
|
match self.replay {
|
|
Replay::PerPoint => {
|
|
// A window holding a native histogram is decided by the mixed-window rules,
|
|
// which the batch kernel -- reading floats alone -- cannot reproduce. Flush
|
|
// what is pending and decide this point on the per-window path.
|
|
//
|
|
// Histograms are retained in time order, so the first one still ahead of the
|
|
// window head decides: a later one cannot be inside this window if this one
|
|
// is not. The check runs only when a batch kernel is set, so a function
|
|
// without one never pays for it.
|
|
let histogram_window = self.batch.is_some()
|
|
&& self.fields.iter().any(|field| {
|
|
field.histograms.get(field.histogram_head).is_some_and(
|
|
|(timestamp, _)| i128::from(*timestamp) <= i128::from(eval_ts),
|
|
)
|
|
});
|
|
if let Some(kernel) = self.batch.as_mut()
|
|
&& !histogram_window
|
|
{
|
|
accumulate(&mut self.pending, state.cursor, eval_ts, &self.fields);
|
|
if self.pending.len() >= EVALUATION_BATCH {
|
|
flush_batch(
|
|
kernel,
|
|
&mut self.pending,
|
|
&self.fields,
|
|
self.window.range_ms,
|
|
&mut self.row,
|
|
&mut self.output,
|
|
#[cfg(test)]
|
|
&mut self.batch_calls,
|
|
)?;
|
|
}
|
|
state.cursor += 1;
|
|
continue;
|
|
}
|
|
if let Some(kernel) = self.batch.as_mut() {
|
|
flush_batch(
|
|
kernel,
|
|
&mut self.pending,
|
|
&self.fields,
|
|
self.window.range_ms,
|
|
&mut self.row,
|
|
&mut self.output,
|
|
#[cfg(test)]
|
|
&mut self.batch_calls,
|
|
)?;
|
|
}
|
|
evaluator.evaluate(eval_ts, eval_ts, &mut self.row)?;
|
|
self.output.push_samples(state.cursor, &self.row)?;
|
|
}
|
|
Replay::Once => {
|
|
evaluator.evaluate(eval_ts, eval_ts, &mut self.row)?;
|
|
for index in 0..grid.len() {
|
|
self.output.push_samples(index, &self.row)?;
|
|
}
|
|
}
|
|
Replay::EveryPoint => {
|
|
for (index, &point) in grid.iter().enumerate() {
|
|
evaluator.evaluate(eval_ts, point, &mut self.row)?;
|
|
self.output.push_samples(index, &self.row)?;
|
|
}
|
|
}
|
|
}
|
|
state.cursor += 1;
|
|
}
|
|
if let Some(kernel) = self.batch.as_mut() {
|
|
flush_batch(
|
|
kernel,
|
|
&mut self.pending,
|
|
&self.fields,
|
|
self.window.range_ms,
|
|
&mut self.row,
|
|
&mut self.output,
|
|
#[cfg(test)]
|
|
&mut self.batch_calls,
|
|
)?;
|
|
}
|
|
// A point is decided once it has been evaluated, once its window can no longer gain a
|
|
// sample, or -- at the end of the series or of what the input has reached -- once no
|
|
// retained sample reaches it. The last case is what keeps a long empty stretch cheap: the
|
|
// loop stops at the first such point instead of walking the rest of the grid, and none of
|
|
// the points behind it can gain a value, so the cursor skips them.
|
|
if empty_tail && matches!(self.replay, Replay::PerPoint) {
|
|
let mut decided = state.cursor;
|
|
if let Some(ready) = ready {
|
|
while decided < points_read && i128::from(grid[decided]) <= ready {
|
|
decided += 1;
|
|
}
|
|
} else {
|
|
decided = points_read;
|
|
}
|
|
state.cursor = decided;
|
|
}
|
|
debug_assert!(
|
|
self.pending.is_empty(),
|
|
"`evaluate` flushes the batch before `advance` rewrites the retained arrays a \
|
|
pending span indexes"
|
|
);
|
|
let mut drains_samples = false;
|
|
for field in &mut self.fields {
|
|
// After the last window, samples are still ingested so that the overflow and
|
|
// histogram checks do not depend on how the series is chunked, but none are kept.
|
|
// An anchored selector can skip past the one point it reads.
|
|
if state.cursor >= points_read {
|
|
field.head = field.timestamps.len();
|
|
field.histogram_head = field.histograms.len();
|
|
}
|
|
drains_samples |= field.head > 0;
|
|
field.advance();
|
|
field.advance_histograms();
|
|
}
|
|
// A drain shifts retained-array indices. Batch flushes do not, so they leave cursors valid.
|
|
if drains_samples {
|
|
self.pending.reset_cursors();
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// The number of leading grid points whose windows are read; an anchored selector reads
|
|
/// only the first.
|
|
fn points_read(&self) -> usize {
|
|
match self.replay {
|
|
Replay::PerPoint => self.grid.spec().len as usize,
|
|
Replay::Once | Replay::EveryPoint => 1,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Computes the result of every field's window at one grid point.
|
|
struct RowEvaluator<'a> {
|
|
fields: &'a [FieldSamples],
|
|
kernel: &'a mut WindowKernel,
|
|
function: Option<RangeFunction>,
|
|
annotations: Option<&'a PromqlAnnotationCollector>,
|
|
range_ms: i64,
|
|
offset_ms: i64,
|
|
modifier: Option<RangeModifier>,
|
|
}
|
|
|
|
impl RowEvaluator<'_> {
|
|
/// Fills `row` with the result over each field's window ending at `window_end`, evaluated
|
|
/// at `eval_ts`.
|
|
fn evaluate(
|
|
&mut self,
|
|
window_end: i64,
|
|
eval_ts: i64,
|
|
row: &mut Vec<Option<GridSample>>,
|
|
) -> Result<()> {
|
|
row.clear();
|
|
// A smoothed window also reads the samples after its end.
|
|
let window_end = window_end + self.modifier.map_or(0, RangeModifier::after_ms);
|
|
for field in self.fields {
|
|
let timestamps = &field.timestamps[field.head..];
|
|
let len = timestamps.partition_point(|×tamp| timestamp <= window_end);
|
|
let floats = (len > 0).then(|| FloatWindow {
|
|
timestamps: ×tamps[..len],
|
|
values: &field.values[field.head..field.head + len],
|
|
counter_resets: field.resets_in(len),
|
|
eval_ts,
|
|
range_ms: self.range_ms,
|
|
});
|
|
let histograms = &field.histograms[field.histogram_head..];
|
|
let histograms = &histograms
|
|
[..histograms.partition_point(|(timestamp, _)| *timestamp <= window_end)];
|
|
row.push(match self.function {
|
|
Some(function) if !histograms.is_empty() => histogram::evaluate(
|
|
function,
|
|
floats,
|
|
histograms,
|
|
eval_ts,
|
|
self.range_ms,
|
|
self.offset_ms,
|
|
&mut *self.kernel,
|
|
self.annotations,
|
|
)?,
|
|
_ => floats.and_then(|window| (self.kernel)(window).map(GridSample::Float)),
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Rejects a modifier on a function Prometheus does not allow it with, as Prometheus rejects the
|
|
/// query (`AnchoredSafeFunctions`, `SmoothedSafeFunctions`), and on evaluated input, which reads
|
|
/// no raw samples around its windows.
|
|
///
|
|
/// `pub(crate)` because it is a rule about the input's schema and the window alone, so the native
|
|
/// route's admission predicate (`native_tree_supported`) asks it rather than a second copy of it,
|
|
/// the way it asks [`fold_offset`].
|
|
pub(crate) fn check_modifier(
|
|
input: &SeriesSchema,
|
|
modifier: RangeModifier,
|
|
function: RangeFunction,
|
|
) -> Result<()> {
|
|
if input.eval_grid.is_some() {
|
|
return Err(DataFusionError::NotImplemented(format!(
|
|
"the {} range modifier over evaluated series",
|
|
modifier.name()
|
|
)));
|
|
}
|
|
let allowed = match modifier {
|
|
RangeModifier::Anchored { .. } => matches!(
|
|
function,
|
|
RangeFunction::Rate
|
|
| RangeFunction::Increase
|
|
| RangeFunction::Delta
|
|
| RangeFunction::Resets
|
|
| RangeFunction::Changes
|
|
),
|
|
RangeModifier::Smoothed { .. } => matches!(
|
|
function,
|
|
RangeFunction::Rate | RangeFunction::Increase | RangeFunction::Delta
|
|
),
|
|
};
|
|
if !allowed {
|
|
let list = match modifier {
|
|
RangeModifier::Anchored { .. } => "changes, delta, increase, rate, resets",
|
|
RangeModifier::Smoothed { .. } => "delta, increase, rate",
|
|
};
|
|
return Err(DataFusionError::Plan(format!(
|
|
"{} modifier can only be used with: {list} - not with {function:?}",
|
|
modifier.name()
|
|
)));
|
|
}
|
|
if modifier.before_ms() < 0 {
|
|
return Err(DataFusionError::Plan(format!(
|
|
"lookback must not be negative, got {} ms",
|
|
modifier.before_ms()
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// The offset samples are shifted by: the selector offset, or for `@` the offset that places
|
|
/// the anchored window at the first grid point.
|
|
///
|
|
/// `pub(crate)` because whether it fits is decided from the grid and the window alone, so the
|
|
/// native route's admission predicate asks the same arithmetic rather than a second copy of it.
|
|
pub(crate) fn fold_offset(grid: GridSpec, window: RangeWindow) -> Result<i64> {
|
|
let Some(anchor) = window.anchor_ms else {
|
|
return Ok(window.offset_ms);
|
|
};
|
|
anchor
|
|
.checked_sub(window.offset_ms)
|
|
.and_then(|end| grid.start_ms.checked_sub(end))
|
|
.ok_or_else(|| {
|
|
DataFusionError::Plan(format!(
|
|
"@ {anchor} ms with offset {} ms is out of range",
|
|
window.offset_ms
|
|
))
|
|
})
|
|
}
|
|
|
|
/// Checks that `predict_linear` can read the instant of every point of `grid` once shifted by
|
|
/// `fold_offset`.
|
|
///
|
|
/// Unlike every other range function, which reads the window a grid point opens alone,
|
|
/// `predict_linear` reads the instant it reports as well, and reports it shifted by the fold
|
|
/// offset. The first and the last point are the ones checked: the shift is the same at every point
|
|
/// and the grid's points are ordered, so what fits at both ends fits between them.
|
|
///
|
|
/// The refusal is arithmetic rather than shape, but it is decided by the grid and the window alone,
|
|
/// which is why the native route's admission predicate reaches it too
|
|
/// (`native_tree_supported`): a query refused only here has to keep the plan it had instead of
|
|
/// failing while its runtime is built.
|
|
pub(crate) fn predict_linear_instants(grid: GridSpec, fold_offset: i64) -> Result<()> {
|
|
let last = i128::from(grid.start_ms) + i128::from(grid.step_ms) * (i128::from(grid.len) - 1);
|
|
for eval_ts in [i128::from(grid.start_ms), last] {
|
|
if i64::try_from(eval_ts + i128::from(fold_offset)).is_err() {
|
|
return Err(DataFusionError::Plan(format!(
|
|
"predict_linear evaluation time {eval_ts} overflows with offset {fold_offset} ms"
|
|
)));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// The inner grid of a subquery read by `window` from the outer grid `outer`: the multiples of
|
|
/// `step_ms` after `start - F - range` up to the last window end, where `F` is the fold offset
|
|
/// of `window`; with `@`, the only window end is `start - F`. `None` when no multiple falls in
|
|
/// the windows.
|
|
pub fn subquery_grid(
|
|
outer: GridSpec,
|
|
window: RangeWindow,
|
|
step_ms: i64,
|
|
) -> Result<Option<GridSpec>> {
|
|
if step_ms <= 0 {
|
|
return Err(DataFusionError::Plan(format!(
|
|
"subquery step must be positive, got {step_ms} ms"
|
|
)));
|
|
}
|
|
let fold_offset = i128::from(fold_offset(outer, window)?);
|
|
let step = i128::from(step_ms);
|
|
let start = i128::from(outer.start_ms);
|
|
let last_eval = match window.anchor_ms {
|
|
Some(_) => start,
|
|
None => start + i128::from(outer.step_ms) * (i128::from(outer.len) - 1),
|
|
};
|
|
let first = ((start - fold_offset - i128::from(window.range_ms)).div_euclid(step) + 1) * step;
|
|
let last = last_eval - fold_offset;
|
|
if first > last {
|
|
return Ok(None);
|
|
}
|
|
let len = u64::try_from((last - first) / step + 1)
|
|
.map_err(|_| DataFusionError::Plan("subquery grid has too many points".into()))?;
|
|
let first = i64::try_from(first).map_err(|_| {
|
|
DataFusionError::Plan(format!("subquery grid start {first} ms is out of range"))
|
|
})?;
|
|
GridSpec::try_new(first, step_ms, len)
|
|
.map(Some)
|
|
.map_err(|error| DataFusionError::External(Box::new(error)))
|
|
}
|