revert(promql): revert #10412 again, unreviewed changes included (#10500)

Revert "revert(promql): restore #10412, undoing the revert (#10497)"

This reverts commit 405c2f9016.
This commit is contained in:
discord9
2026-11-18 12:13:40 +08:00
committed by GitHub
parent 405c2f9016
commit 183d1d42f7
44 changed files with 1944 additions and 7686 deletions
Generated
-1
View File
@@ -12087,7 +12087,6 @@ dependencies = [
"snafu 0.8.6",
"sql",
"sqlparser",
"stacker",
"store-api",
"substrait 1.4.0-alpha.0",
"table",
+17 -59
View File
@@ -148,41 +148,19 @@ impl EvalGrid {
self.timestamps.get_or_try_init(|| {
let len = usize::try_from(self.spec.len)
.map_err(|_| invalid_series("grid length overflows usize"))?;
Ok(self.gather_timestamps(0, len)?.into())
let mut values = Vec::new();
values
.try_reserve_exact(len)
.map_err(|_| invalid_series("grid timestamp allocation failed"))?;
for index in 0..len {
let timestamp =
(self.spec.start_ms as i128) + (index as i128) * (self.spec.step_ms as i128);
values.push(timestamp as i64);
}
Ok(values.into())
})
}
/// Whether the whole timestamp array exists already.
///
/// A consumer that pays for the array -- an execution reserving what it is about to allocate --
/// asks this before it materializes a grid: a grid that is already materialized was charged to
/// whoever materialized it, and a second consumer must not charge it again.
pub fn is_materialized(&self) -> bool {
self.timestamps.get().is_some()
}
/// Returns the timestamps of the positions `start..start + len` as an array of their own,
/// leaving the grid unmaterialized.
///
/// Every value is [`Self::timestamp_at`]'s, in position order, so gathering a window of the
/// grid costs its own length and not the length of the grid it is cut from.
fn gather_timestamps(&self, start: usize, len: usize) -> Result<Vec<i64>> {
let mut values = Vec::new();
values
.try_reserve_exact(len)
.map_err(|_| invalid_series("grid timestamp allocation failed"))?;
let end = start.checked_add(len).context(InvalidSeriesSnafu {
reason: "grid range overflow".to_string(),
})?;
for position in start..end {
let Some(timestamp) = self.timestamp_at(position) else {
return invalid("grid range exceeds its grid");
};
values.push(timestamp);
}
Ok(values)
}
fn timestamp_at(&self, position: usize) -> Option<i64> {
if position as u64 >= self.spec.len {
return None;
@@ -233,22 +211,9 @@ impl GridRange {
pub fn is_empty(&self) -> bool {
self.len == 0
}
/// Returns the timestamps of the positions this range names.
///
/// A range covering the whole grid shares the grid's array, which that grid has to materialize
/// for any reader of all of it. A range cut out of a larger grid computes the `len` timestamps
/// it names instead, each one [`EvalGrid::timestamp_at`]'s for its position: reading a window
/// out of a large grid costs the window's rows, and leaves a deferred grid deferred.
/// Returns this range as a zero-copy Arrow buffer slice.
pub fn timestamps(&self) -> Result<ScalarBuffer<i64>> {
if self.covers_grid() {
return Ok(self.grid.timestamps()?.clone());
}
Ok(self.grid.gather_timestamps(self.start, self.len)?.into())
}
/// Whether this range names every position of its grid.
fn covers_grid(&self) -> bool {
self.start == 0 && u64::try_from(self.len).is_ok_and(|len| len == self.grid.spec.len)
Ok(self.grid.timestamps()?.slice(self.start, self.len))
}
}
@@ -844,20 +809,13 @@ impl TimeAxis {
/// Returns the timestamps of the carried rows, in compact-row order.
///
/// An axis covering the whole grid lends the grid's array; a grid axis holding fewer positions
/// than its grid computes the rows it carries and lends nothing, so gathering a window out of
/// a large grid costs the window and leaves a deferred grid deferred. An axis of several runs,
/// and a chunked explicit axis, allocate one array of the carried rows.
/// A single-run grid axis lends its buffer without copying; an axis of several runs, and a
/// chunked explicit axis, allocate one array of the carried rows.
pub fn timestamps(&self) -> Result<Cow<'_, [i64]>> {
match self {
Self::Grid(range) => {
if !range.covers_grid() {
return Ok(Cow::Owned(
range.grid.gather_timestamps(range.start(), range.len())?,
));
}
Ok(Cow::Borrowed(&range.grid.timestamps()?[..]))
}
Self::Grid(range) => Ok(Cow::Borrowed(
&range.grid.timestamps()?[range.start()..range.start() + range.len()],
)),
Self::GridRanges(ranges) => {
let mut timestamps = Vec::with_capacity(ranges.len());
for (position, _) in self.positions() {
+5 -70
View File
@@ -118,67 +118,11 @@ fn deferred_grid_allocation_failure_is_reported_and_not_cached() {
Arc::new(EvalGrid::try_deferred(GridSpec::try_new(i64::MIN, 1, len).unwrap()).unwrap());
assert!(grid.timestamps().is_err());
assert!(grid.timestamps.get().is_none());
// A range naming every point of that grid asks for the array that does not fit, and the
// failure is still the series error rather than an abort.
let whole = GridRange::try_new(grid.clone(), 0, len as usize).unwrap();
assert!(whole.timestamps().is_err());
assert!(TimeAxis::Grid(whole).timestamps().is_err());
let range = GridRange::try_new(grid.clone(), 0, 1).unwrap();
assert!(range.timestamps().is_err());
assert!(TimeAxis::Grid(range).timestamps().is_err());
assert!(grid.timestamps().is_err());
assert!(grid.timestamps.get().is_none());
// A range that is nearly the whole grid gathers its own rows, and its allocation fails too:
// the gathered array is the range's, so the failure is reported rather than aborted.
let nearly = GridRange::try_new(grid.clone(), 0, len as usize - 1).unwrap();
assert!(nearly.timestamps().is_err());
assert!(TimeAxis::Grid(nearly).timestamps().is_err());
assert!(grid.timestamps.get().is_none());
}
/// A window cut out of a large grid computes the rows it names instead of materializing the grid.
///
/// The gathered values are the grid's own points, so a subrange reads the same timestamps it did
/// when it was a slice of the whole array; what changes is the cost, which is the window's rows
/// and no longer the grid's.
#[test]
fn subrange_timestamps_gather_only_the_rows_they_name() {
let len = (isize::MAX as u64 / std::mem::size_of::<i64>() as u64) + 1;
let grid = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(-5, 2, len).unwrap()).unwrap());
let range = GridRange::try_new(grid.clone(), 7, 3).unwrap();
assert!(!grid.is_materialized());
let gathered = range.timestamps().unwrap();
assert_eq!(&gathered[..], &[9, 11, 13]);
let axis = TimeAxis::Grid(range);
let axis_timestamps = axis.timestamps().unwrap();
assert_eq!(axis_timestamps.as_ref(), &[9, 11, 13]);
// Every gathered point is what the axis answers for the row holding it.
for (row, timestamp) in axis_timestamps.iter().enumerate() {
assert_eq!(*timestamp, axis.timestamp(row).unwrap());
}
assert!(
!grid.is_materialized(),
"the subrange materialized its grid"
);
// A range naming the whole grid still shares the grid's one array.
let small = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(10, 10, 4).unwrap()).unwrap());
let all = GridRange::try_new(small.clone(), 0, 4).unwrap();
assert!(!small.is_materialized());
let shared = all.timestamps().unwrap();
assert!(
small.is_materialized(),
"the whole-grid range did not materialize the grid it shares"
);
assert_eq!(shared.as_ptr(), small.timestamps().unwrap().as_ptr());
assert_eq!(
TimeAxis::Grid(all).timestamps().unwrap().as_ref(),
&[10, 20, 30, 40]
);
// An empty range at the end of a grid has no row to gather and materializes nothing.
let grid = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(0, 1, 3).unwrap()).unwrap());
let empty = GridRange::try_new(grid.clone(), 3, 0).unwrap();
assert!(empty.timestamps().unwrap().is_empty());
assert!(TimeAxis::Grid(empty).timestamps().unwrap().is_empty());
assert!(!grid.is_materialized());
}
#[test]
@@ -223,20 +167,11 @@ fn grid_materialization_revalidates_public_specs_and_shares_ranges_by_value() {
let b = GridRange::try_new(Arc::new(EvalGrid::try_new(grid.spec()).unwrap()), 1, 3).unwrap();
assert_eq!(a, b);
let values = a.timestamps().unwrap();
// A subrange computes its own rows rather than slicing the grid's array, which it no longer
// shares: what it answers is still the grid's points, one per position it names.
assert_ne!(
assert_eq!(
values.as_ptr(),
grid.timestamps().unwrap().as_ptr().wrapping_add(1),
"the subrange borrowed the grid's array"
grid.timestamps().unwrap().as_ptr().wrapping_add(1)
);
assert_eq!(&values[..], &[-2, 1, 4]);
let whole = GridRange::try_new(grid.clone(), 0, 5).unwrap();
assert_eq!(
whole.timestamps().unwrap().as_ptr(),
grid.timestamps().unwrap().as_ptr(),
"a whole-grid range copied the array it could share"
);
assert!(GridRange::try_new(grid.clone(), 5, 0).unwrap().is_empty());
assert!(GridRange::try_new(grid.clone(), 4, 2).is_err());
assert!(GridRange::try_new(grid, usize::MAX, 1).is_err());
+16 -145
View File
@@ -226,52 +226,6 @@ pub(crate) async fn reconcile_series_indexes_with_mode(
Ok(stats)
}
/// How many sources one samples build reads at once. The writer bounds blocks and output sizes on
/// its own, so this only bounds the memory one pinned source group holds.
const SAMPLES_SOURCES_PER_BUILD: usize = 64;
/// Splits uncovered sources into samples build groups, by UTC-day window and then by SST level.
///
/// A samples artifact records the files it was built from and the days each of them contributed, so
/// a group is an identity, not a batch: every output of one group names the same sources, and
/// coverage is read back per group. Cutting the sources by file id alone - the previous behaviour -
/// put every level and every window into the same arbitrary chunks, so one new file re-cut them and
/// the days of unrelated levels were rebuilt. Grouping by window and level keeps a group's boundary
/// where the data puts it: a file of another level or another window joins its own group, and the
/// outputs already published for a group stay reusable when a source is added beside it.
///
/// Windows sort before levels so the days of one window are built next to each other, and the files
/// inside a group are ordered by id so a build reads them in the same order every time.
fn sample_source_groups(files: &[FileHandle]) -> Vec<Vec<FileHandle>> {
use std::collections::BTreeMap;
use crate::series_index::builder::sample_day;
let mut groups = BTreeMap::<(i64, i64, u8), Vec<FileHandle>>::new();
for file in files {
// The span the file covers is the window it writes into. A file holding several days is a
// window of its own here and still reports the days it actually filled.
let (start, end) = file.time_range();
let (start, end) = (sample_day(start), sample_day(end));
groups
.entry((start.min(end), start.max(end), file.level()))
.or_default()
.push(file.clone());
}
groups
.into_values()
.map(|mut group| {
group.sort_unstable_by(|left, right| {
left.file_id()
.file_id()
.as_bytes()
.cmp(right.file_id().file_id().as_bytes())
});
group
})
.collect()
}
/// Builds a changed snapshot, retaining reusable indexes and removing obsolete coverage.
/// Returns `None` when no range or series indexes were added or removed.
#[allow(clippy::too_many_arguments)]
@@ -504,27 +458,26 @@ async fn build_index_version(
.values()
.flat_map(|handle| handle.entry().source_file_ids.iter().copied())
.collect();
let uncovered: Vec<_> = files
let mut uncovered: Vec<_> = files
.iter()
.filter(|file| !covered.contains(&file.file_id().file_id()))
.cloned()
.collect();
for sources in sample_source_groups(&uncovered) {
for sources in sources.chunks(SAMPLES_SOURCES_PER_BUILD) {
let handles = crate::series_index::builder::build_sample_windows(
store,
region,
metadata,
sources,
backend.as_ref(),
purger,
)
.await?;
for handle in handles {
unpublished.1.push(handle.clone());
stats.built_samples += 1;
daily.insert(handle.entry().file_id, handle);
}
uncovered.sort_unstable_by_key(|file| file.file_id().file_id().to_string());
for sources in uncovered.chunks(64) {
let handles = crate::series_index::builder::build_sample_windows(
store,
region,
metadata,
sources,
backend.as_ref(),
purger,
)
.await?;
for handle in handles {
unpublished.1.push(handle.clone());
stats.built_samples += 1;
daily.insert(handle.entry().file_id, handle);
}
}
stats.removed_samples += samples.keys().filter(|id| !daily.contains_key(id)).count();
@@ -740,95 +693,13 @@ fn publish_index_version(region: &MitoRegionRef, next: Arc<SeriesIndexVersion>)
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use common_time::Timestamp;
use object_store::services::Memory;
use super::*;
use crate::series_index::builder::sample_day;
use crate::series_index::catalog::{RangeIndexEntry, SeriesIndexEntry};
use crate::series_index::purger::series_index_channel;
use crate::sst::file::{FileHandle, FileMeta};
use crate::sst::file_purger::NoopFilePurger;
/// A source file covering whole UTC days `start_day..=end_day` at `level`.
fn samples_source(id: u8, level: u8, start_day: i64, end_day: i64) -> FileHandle {
const DAY_MS: i64 = 86_400_000;
FileHandle::new(
FileMeta {
file_id: FileId::from_bytes([id; 16]),
time_range: (
Timestamp::new_millisecond(start_day * DAY_MS),
Timestamp::new_millisecond(end_day * DAY_MS + DAY_MS - 1),
),
level,
..Default::default()
},
Arc::new(NoopFilePurger),
)
}
/// Sources are grouped by the window they cover and by their level, not cut by file id.
///
/// Every group is one window of one level, so a file that joins a level later cannot move the
/// boundary of a group built for another level, and the group's days are the days it covers.
#[test]
fn samples_sources_are_grouped_by_window_and_level() {
let mut files = vec![
samples_source(1, 0, 0, 0),
samples_source(2, 0, 0, 0),
samples_source(3, 1, 0, 0),
samples_source(4, 0, 5, 5),
samples_source(5, 1, 5, 7),
samples_source(6, 0, 5, 5),
];
// Grouping must not depend on the ids or on the order the files arrive in.
files.rotate_left(2);
let groups = sample_source_groups(&files);
let describe = |group: &[FileHandle]| {
let days: Vec<(i64, i64)> = group
.iter()
.map(|file| {
let (start, end) = file.time_range();
(sample_day(start), sample_day(end))
})
.collect();
(group[0].level(), days)
};
assert_eq!(
vec![
(0, vec![(0, 0), (0, 0)]),
(1, vec![(0, 0)]),
(0, vec![(5, 5), (5, 5)]),
(1, vec![(5, 7)]),
],
groups
.iter()
.map(|group| describe(group))
.collect::<Vec<_>>(),
"one window of one level per group, ordered by window then level"
);
// Every source is placed exactly once, and no group mixes a level or a window.
let mut grouped: Vec<_> = groups
.iter()
.flatten()
.map(|file| file.file_id().file_id().as_bytes().to_vec())
.collect();
let mut sources: Vec<_> = files
.iter()
.map(|file| file.file_id().file_id().as_bytes().to_vec())
.collect();
grouped.sort_unstable();
sources.sort_unstable();
assert_eq!(sources, grouped);
for group in &groups {
let (level, days) = describe(group);
assert!(group.iter().all(|file| file.level() == level));
assert!(days.windows(2).all(|pair| pair[0] == pair[1]));
}
}
#[rstest::rstest]
fn test_prune_index_version(
+146 -232
View File
@@ -123,24 +123,6 @@ pub(crate) enum HistogramSamples {
Folded,
}
impl AggregateOp {
/// The words of `f64` one field's state of this aggregation keeps for one grid point.
///
/// The last word is the presence word: the op's own count where its state carries one, so a
/// zero count reads as unseen, and a `0`/`1` flag where it does not. The words before it are
/// the components the op folds, in [`Self::state_fields`] order. A state therefore costs what
/// its own operation needs instead of what the widest operation needs: a block holds
/// `BLOCK_POINTS * fields * state_words * size_of::<f64>()` bytes however wide the op is.
pub(crate) const fn state_words(self) -> usize {
match self {
Self::Count | Self::Group => 1,
Self::Min | Self::Max => 2,
Self::Sum | Self::Stddev | Self::Stdvar => 3,
Self::Avg => 5,
}
}
}
/// What `op` does with a native histogram sample.
pub(crate) fn histogram_samples(op: AggregateOp) -> HistogramSamples {
match op {
@@ -236,45 +218,16 @@ impl FloatAggregator {
) -> Result<()> {
validate_aggregate_series(&self.schema, series, self.op)?;
let existing_bytes = self.state_bytes();
let group_capacity = self.groups.capacity();
let map_capacity = self.group_index.capacity();
// One admission for everything the fold can allocate before it writes a sample: the copy of
// this series' metric the grouping makes, the group's own copy of those labels and its
// key, the labels vector, and the group vector and index that may grow. What only a new
// group pays for is admitted whether or not this series opens one, so a chunk costs one
// pool call here instead of one per step, and a refusal still arrives before any of it is
// allocated.
//
// The grouping copy, the second workspace the copy is grouped into, and the group's
// retained labels and key are each a copy or a subset of this series' labels, so three
// copies of the one workspace bound all of them.
let label_workspace = grouping_workspace_bytes(&series.metric);
budget(
existing_bytes
.saturating_add(label_workspace.saturating_mul(3))
.saturating_add(pending_bytes(self.schema.fields.len()))
.saturating_add(
self.labels
.len()
.saturating_mul(std::mem::size_of::<&str>()),
)
// A realloc that cannot grow in place keeps both buffers alive while it copies,
// so a growing step admits the new buffer beside the old one, which the retained
// state already holds. The map is admitted the same way.
.saturating_add(if self.groups.len() == group_capacity {
group_capacity
.saturating_mul(2)
.max(4)
.saturating_mul(size_of::<Group>())
} else {
0
})
.saturating_add(if self.group_index.len() == map_capacity {
hash_map_backing_bytes(next_hash_capacity(map_capacity))
} else {
0
}),
)?;
let grouping_workspace = grouping_workspace_bytes(&series.metric);
let mut workspace_bytes =
grouping_workspace.saturating_add(pending_bytes(self.schema.fields.len()));
budget(existing_bytes.saturating_add(workspace_bytes))?;
workspace_bytes = workspace_bytes.saturating_add(
self.labels
.len()
.saturating_mul(std::mem::size_of::<&str>()),
);
budget(existing_bytes.saturating_add(workspace_bytes))?;
let labels: Vec<&str> = self.labels.iter().map(|label| &**label).collect();
let grouping = if self.by {
Grouping::By(&labels)
@@ -282,12 +235,51 @@ impl FloatAggregator {
Grouping::Without(&labels)
};
let metric = grouping.apply(&series.metric)?;
let key_bytes = metric
.tags()
.len()
.saturating_add(1)
.saturating_mul(std::mem::size_of::<(Arc<str>, Arc<str>)>());
workspace_bytes = workspace_bytes
.saturating_add(grouping_workspace_bytes(&metric))
.saturating_add(key_bytes);
budget(existing_bytes.saturating_add(workspace_bytes))?;
let key = label_key(&metric);
let key_retained = key
.capacity()
.saturating_mul(size_of::<(Arc<str>, Arc<str>)>());
let new_group = !self.group_index.contains_key(&key);
let group_capacity = self.groups.capacity();
let map_capacity = self.group_index.capacity();
let old_map_backing = hash_map_backing_bytes(map_capacity);
if new_group {
let next_group_capacity = if self.groups.len() == group_capacity {
group_capacity.saturating_mul(2).max(4)
} else {
group_capacity
};
let group_realloc_bytes = if next_group_capacity > group_capacity {
next_group_capacity.saturating_mul(size_of::<Group>())
} else {
0
};
budget(
existing_bytes
.saturating_add(metric_bytes(&metric))
.saturating_add(key_bytes)
// A realloc that cannot grow in place keeps both buffers alive while it
// copies, so a growing step admits the old buffer beside the new one --
// the same reason the map below admits both tables. The old buffer alone
// is already in `existing_bytes`; what is added here is the new one.
.saturating_add(group_realloc_bytes)
.saturating_add(if self.group_index.len() == map_capacity {
hash_map_backing_bytes(next_hash_capacity(map_capacity))
} else {
0
})
.saturating_add(workspace_bytes),
)?;
}
let group_capacity_growth = if new_group && self.groups.len() == group_capacity {
group_capacity.saturating_mul(2).max(4)
} else {
@@ -383,8 +375,7 @@ impl FloatAggregator {
.len()
.saturating_mul(BLOCK_POINTS)
.saturating_mul(self.schema.fields.len())
.saturating_mul(self.op.state_words())
.saturating_mul(std::mem::size_of::<f64>()),
.saturating_mul(std::mem::size_of::<FloatState>()),
)
.saturating_add(btree_backing_bytes(group.blocks.len()))
.saturating_add(histogram_backing_bytes(group.histograms.len()))
@@ -468,8 +459,6 @@ fn finish_groups_budgeted(
DataFusionError::Execution(format!("cannot reserve aggregate row: {error}"))
})?;
let mut completed_output_bytes = 0usize;
let words = op.state_words();
let point_words = width.saturating_mul(words);
for group in groups {
// A group that folded histogram samples answers histogram columns; one that folded floats
// keeps the float path below, so a group without a single histogram payload is built
@@ -498,7 +487,7 @@ fn finish_groups_budgeted(
}
// One builder serves every group, so each result series starts explicitly.
builder.begin_series();
let (rows, runs) = group_rows_runs(&group, width, op);
let (rows, runs) = group_rows_runs(&group, width);
let finish_bytes = finish_working_bytes(rows, runs, width);
budget(
state
@@ -510,16 +499,12 @@ fn finish_groups_budgeted(
builder.reserve_rows(rows)?;
for (&block, states) in &group.blocks {
// The last block can extend past the grid; only points with a sample are pushed.
for (offset, point) in states.chunks_exact(point_words).enumerate() {
if !point.chunks_exact(words).any(|state| state_seen(state, op)) {
for (offset, point) in states.chunks_exact(width).enumerate() {
if !point.iter().any(|state| state.seen) {
continue;
}
row.clear();
row.extend(
point
.chunks_exact(words)
.map(|state| state_finish(state, op)),
);
row.extend(point.iter().map(|state| state.finish(op)));
builder.push(block * BLOCK_POINTS + offset, &row)?;
}
}
@@ -556,12 +541,11 @@ fn finish_histogram_group(
// A field's column is one type. The fold already refuses a histogram where it had folded a
// float at that position; a float folded after a histogram is caught here, where the column
// type is chosen for the whole group.
let words = op.state_words();
if group.blocks.values().any(|states| {
states
.chunks_exact(words)
.any(|state| state_seen(state, op))
}) {
if group
.blocks
.values()
.any(|states| states.iter().any(|state| state.seen))
{
return Err(DataFusionError::Execution(
"a native aggregate cannot mix float and histogram samples in one field".into(),
));
@@ -713,18 +697,16 @@ fn finish_groups(
finish_groups_budgeted(schema, op, groups, 0, |_| Ok(()))
}
fn group_rows_runs(group: &Group, width: usize, op: AggregateOp) -> (usize, usize) {
fn group_rows_runs(group: &Group, width: usize) -> (usize, usize) {
if width == 0 {
return (0, 0);
}
let words = op.state_words();
let point_words = width * words;
let mut rows = 0usize;
let mut runs = 0usize;
let mut previous: Option<usize> = None;
for (&block, states) in &group.blocks {
for (offset, point) in states.chunks_exact(point_words).enumerate() {
if point.chunks_exact(words).any(|state| state_seen(state, op)) {
for (offset, point) in states.chunks_exact(width).enumerate() {
if point.iter().any(|state| state.seen) {
let index = block.saturating_mul(BLOCK_POINTS).saturating_add(offset);
rows = rows.saturating_add(1);
if previous.is_none_or(|previous| previous.checked_add(1) != Some(index)) {
@@ -1111,7 +1093,7 @@ fn next_hash_capacity(capacity: usize) -> usize {
/// words; the leaf form is smaller than that. The header allowance is generous on purpose, so the
/// bound does not depend on how a toolchain lays the node out.
fn btree_node_bytes() -> usize {
11 * size_of::<(usize, Box<[f64]>)>() + 16 * size_of::<usize>()
11 * size_of::<(usize, Box<[FloatState]>)>() + 16 * size_of::<usize>()
}
/// An upper bound on the bytes a B-tree of `entries` holds.
@@ -1166,21 +1148,11 @@ fn label_key(metric: &SeriesLabels) -> LabelKey {
/// input never expands to the full grid and dense input costs one allocation per block.
const BLOCK_POINTS: usize = 64;
/// Bytes one allocated block keeps: a state per point per field, [`AggregateOp::state_words`]
/// words of `f64` each. The fold, the merge and the recount all size a block through this, so what
/// is admitted and what is allocated cannot drift apart.
fn block_bytes(width: usize, op: AggregateOp) -> usize {
BLOCK_POINTS
.saturating_mul(width)
.saturating_mul(op.state_words())
.saturating_mul(size_of::<f64>())
}
struct Group {
metric: SeriesLabels,
/// Keyed by absolute grid index / `BLOCK_POINTS`; each block holds one state per field for
/// each of its points, point by point, each state as many `f64` words as the op folds.
blocks: BTreeMap<usize, Box<[f64]>>,
/// each of its points, point by point.
blocks: BTreeMap<usize, Box<[FloatState]>>,
/// The histogram states of this group, keyed by absolute grid position and field.
///
/// Only an aggregate that folds a payload allocates one, and only where it saw a histogram
@@ -1255,20 +1227,6 @@ struct FieldFold {
op: AggregateOp,
}
impl FieldFold {
/// Words one point of this fold's block keeps: every field's state, side by side.
fn point_words(self) -> usize {
self.width.saturating_mul(self.op.state_words())
}
/// Where this field's state of the point at `offset` starts inside a block.
fn state_start(self, offset: usize) -> usize {
offset
.saturating_mul(self.point_words())
.saturating_add(self.field_index.saturating_mul(self.op.state_words()))
}
}
impl Group {
/// Folds one series' float samples into this group's blocks, one field at a time.
///
@@ -1353,15 +1311,14 @@ impl Group {
let block = index / BLOCK_POINTS;
let (states, admitted) = self.block_states(block, fold, budget, retained)?;
retained = admitted;
let words = fold.op.state_words();
let start = fold.state_start(index % BLOCK_POINTS);
FloatState::new(&mut states[start..start + words], fold.op).update(sample);
states[(index % BLOCK_POINTS) * fold.width + fold.field_index]
.update(fold.op, sample);
while let Some(&(at, sample)) = samples.peek()
&& at / BLOCK_POINTS == block
{
samples.next();
let start = fold.state_start(at % BLOCK_POINTS);
FloatState::new(&mut states[start..start + words], fold.op).update(sample);
states[(at % BLOCK_POINTS) * fold.width + fold.field_index]
.update(fold.op, sample);
}
}
}
@@ -1379,11 +1336,13 @@ impl Group {
fold: FieldFold,
budget: &mut impl FnMut(usize) -> Result<()>,
retained: usize,
) -> Result<(&'a mut [f64], usize)> {
) -> Result<(&'a mut [FloatState], usize)> {
let new_block = !self.blocks.contains_key(&block);
let (states_bytes, backing_delta) = if new_block {
let (block_bytes, backing_delta) = if new_block {
(
block_bytes(fold.width, fold.op),
BLOCK_POINTS
.saturating_mul(fold.width)
.saturating_mul(size_of::<FloatState>()),
btree_backing_bytes(self.blocks.len() + 1)
.saturating_sub(btree_backing_bytes(self.blocks.len())),
)
@@ -1392,14 +1351,13 @@ impl Group {
};
let retained = retained
.saturating_add(backing_delta)
.saturating_add(states_bytes);
.saturating_add(block_bytes);
if new_block {
budget(retained)?;
}
let states = self
.blocks
.entry(block)
.or_insert_with(|| vec![0.0; BLOCK_POINTS * fold.point_words()].into_boxed_slice());
let states = self.blocks.entry(block).or_insert_with(|| {
vec![FloatState::default(); BLOCK_POINTS * fold.width].into_boxed_slice()
});
Ok((&mut states[..], retained))
}
}
@@ -1429,10 +1387,8 @@ impl Group {
// A field's column is either float or histogram, so a position that already folded a float
// cannot also fold a histogram. The other order is caught when the fold is finished, which
// is where the column type is chosen.
let words = fold.op.state_words();
let start = fold.state_start(position % BLOCK_POINTS);
if let Some(states) = self.blocks.get(&(position / BLOCK_POINTS))
&& state_seen(&states[start..start + words], fold.op)
&& states[(position % BLOCK_POINTS) * fold.width + fold.field_index].seen
{
return Err(DataFusionError::Execution(
"a native aggregate cannot mix float and histogram samples in one field".into(),
@@ -1863,137 +1819,95 @@ fn histogram_backing_bytes(entries: usize) -> usize {
entries.div_ceil(5).saturating_mul(histogram_node_bytes())
}
/// The float part of Prometheus' `groupedAggregation`, as a view of one state's words.
///
/// A state is [`AggregateOp::state_words`] words of the block it sits in: the components the op
/// folds, then the presence word. The view reads and writes that layout in place, so folding
/// allocates nothing per state and an op that keeps less memory keeps less.
struct FloatState<'a> {
words: &'a mut [f64],
op: AggregateOp,
/// The float part of Prometheus' `groupedAggregation`.
#[derive(Clone, Copy, Default)]
struct FloatState {
seen: bool,
value: f64,
mean: f64,
compensation: f64,
count: f64,
incremental_mean: bool,
}
/// Whether the state in `words` has seen a sample: its presence word is set.
///
/// The presence word is the last one, which is the op's count where its state carries one - a
/// state that has folded nothing has no count - and a `0`/`1` flag where it does not.
fn state_seen(words: &[f64], op: AggregateOp) -> bool {
words[op.state_words() - 1] != 0.0
}
/// The value the state in `words` finishes to, or `None` if it has seen no sample.
fn state_finish(words: &[f64], op: AggregateOp) -> Option<f64> {
if !state_seen(words, op) {
return None;
}
Some(match op {
// `value`, its compensation, then the presence flag.
AggregateOp::Sum => words[0] + words[1],
// `value`, `compensation`, `mean`, the incremental flag, then the count.
AggregateOp::Avg if words[3] != 0.0 => words[2] + words[1],
AggregateOp::Avg => words[0] / words[4] + words[1] / words[4],
AggregateOp::Count => words[0],
AggregateOp::Min | AggregateOp::Max => words[0],
AggregateOp::Group => 1.0,
// `value`, `mean`, then the count.
AggregateOp::Stdvar => words[0] / words[2],
AggregateOp::Stddev => (words[0] / words[2]).sqrt(),
})
}
impl<'a> FloatState<'a> {
/// The state at `words`, which is `op`'s layout: [`AggregateOp::state_words`] words.
fn new(words: &'a mut [f64], op: AggregateOp) -> Self {
debug_assert_eq!(words.len(), op.state_words());
Self { words, op }
}
/// Folds one sample in, initializing the state if it had seen none.
fn update(self, sample: f64) {
let (op, words) = (self.op, self.words);
if !state_seen(words, op) {
match op {
// The count is the presence word.
AggregateOp::Count => words[0] = 1.0,
// Presence alone: `group` answers 1 for any kind of sample, NaN included, so the
// sample itself is not kept - a zero sample would read as unseen.
AggregateOp::Group => words[0] = 1.0,
// `value`, then the presence flag.
AggregateOp::Min | AggregateOp::Max => {
words[0] = sample;
words[1] = 1.0;
}
// `value`, its compensation, then the presence flag.
AggregateOp::Sum => {
words[0] = sample;
words[1] = 0.0;
words[2] = 1.0;
}
// The running sum of squared deviations, which Prometheus starts at NaN for a
// non-finite first sample; then `mean` and the count that says it is seen.
AggregateOp::Stddev | AggregateOp::Stdvar => {
words[0] = if sample.is_finite() { 0.0 } else { f64::NAN };
words[1] = sample;
words[2] = 1.0;
}
// `value`, `compensation`, `mean`, the incremental flag, then the count.
AggregateOp::Avg => {
words[0] = sample;
words[1] = 0.0;
words[2] = sample;
words[3] = 0.0;
words[4] = 1.0;
}
}
impl FloatState {
fn update(&mut self, op: AggregateOp, sample: f64) {
if !self.seen {
*self = Self {
seen: true,
value: match op {
// The running sum of squared deviations; Prometheus starts it at NaN for a
// non-finite first sample.
AggregateOp::Stddev | AggregateOp::Stdvar if !sample.is_finite() => f64::NAN,
AggregateOp::Stddev | AggregateOp::Stdvar => 0.0,
_ => sample,
},
mean: sample,
count: 1.0,
..Self::default()
};
return;
}
match op {
AggregateOp::Sum => {
(words[0], words[1]) = kahan_inc(sample, words[0], words[1]);
(self.value, self.compensation) = kahan_inc(sample, self.value, self.compensation);
}
AggregateOp::Avg => update_avg(words, sample),
AggregateOp::Count => words[0] += 1.0,
AggregateOp::Avg => self.update_avg(sample),
AggregateOp::Count => self.count += 1.0,
AggregateOp::Max => {
if words[0] < sample || words[0].is_nan() {
words[0] = sample;
if self.value < sample || self.value.is_nan() {
self.value = sample;
}
}
AggregateOp::Min => {
if words[0] > sample || words[0].is_nan() {
words[0] = sample;
if self.value > sample || self.value.is_nan() {
self.value = sample;
}
}
AggregateOp::Group => {}
// Welford's update, uncompensated as in Prometheus.
AggregateOp::Stddev | AggregateOp::Stdvar => {
let count = words[2] + 1.0;
let delta = sample - words[1];
words[1] += delta / count;
words[0] += delta * (sample - words[1]);
words[2] = count;
self.count += 1.0;
let delta = sample - self.mean;
self.mean += delta / self.count;
self.value += delta * (sample - self.mean);
}
}
}
}
/// `avg`'s update: sums directly until the running sum would overflow, then switches to an
/// incremental mean, whose flag is the fourth word.
fn update_avg(words: &mut [f64], sample: f64) {
let count = words[4] + 1.0;
words[4] = count;
if words[3] == 0.0 {
let (value, compensation) = kahan_inc(sample, words[0], words[1]);
if !value.is_infinite() {
words[0] = value;
words[1] = compensation;
return;
/// Sums directly until the running sum would overflow, then switches to an incremental mean.
fn update_avg(&mut self, sample: f64) {
self.count += 1.0;
if !self.incremental_mean {
let (value, compensation) = kahan_inc(sample, self.value, self.compensation);
if !value.is_infinite() {
(self.value, self.compensation) = (value, compensation);
return;
}
self.incremental_mean = true;
self.mean = self.value / (self.count - 1.0);
self.compensation /= self.count - 1.0;
}
words[3] = 1.0;
words[2] = words[0] / (count - 1.0);
words[1] /= count - 1.0;
let q = (self.count - 1.0) / self.count;
(self.mean, self.compensation) =
kahan_inc(sample / self.count, q * self.mean, q * self.compensation);
}
fn finish(self, op: AggregateOp) -> Option<f64> {
if !self.seen {
return None;
}
Some(match op {
AggregateOp::Sum => self.value + self.compensation,
AggregateOp::Avg if self.incremental_mean => self.mean + self.compensation,
AggregateOp::Avg => self.value / self.count + self.compensation / self.count,
AggregateOp::Count => self.count,
AggregateOp::Min | AggregateOp::Max => self.value,
AggregateOp::Group => 1.0,
AggregateOp::Stdvar => self.value / self.count,
AggregateOp::Stddev => (self.value / self.count).sqrt(),
})
}
let q = (count - 1.0) / count;
(words[2], words[1]) = kahan_inc(sample / count, q * words[2], q * words[1]);
}
/// Prometheus `kahansum.Inc`, with the Neumaier swap.
+90 -168
View File
@@ -42,9 +42,9 @@ use datatypes::series::time::{GridSpan, TimeAxis};
use crate::aggregate::{
AggregateOp, BLOCK_POINTS, Counts, FloatAggregator, FloatState, Group, HistogramAccumulator,
HistogramEvidence, LabelKey, accumulator_bytes, block_bytes, btree_backing_bytes, count_series,
HistogramEvidence, LabelKey, accumulator_bytes, btree_backing_bytes, count_series,
evidence_summary, finish_groups, finish_groups_budgeted, hash_map_backing_bytes,
histogram_backing_bytes, kahan_inc, label_key, metric_bytes, next_hash_capacity, state_seen,
histogram_backing_bytes, kahan_inc, label_key, metric_bytes, next_hash_capacity,
};
use crate::map::validate_float_series;
@@ -127,7 +127,7 @@ impl FloatAggregator {
// export builds two one-row Arrow batches per point, which the chunk they are handed
// to is charged for only once that chunk exists.
let (points, payloads) = if group.histograms.is_empty() {
(seen_points(group, width, self.op), 0)
(seen_points(group, width), 0)
} else {
let payloads = group
.histograms
@@ -165,21 +165,18 @@ impl FloatAggregator {
DataFusionError::Execution("aggregate state is indexed on an evaluated grid".into())
})?;
let grid_len = grid.spec().len as usize;
let op = self.op;
let words = op.state_words();
let point_words = width.saturating_mul(words);
let mut groups: Vec<StateSeries> = Vec::with_capacity(self.groups.len());
for group in self.groups {
let points = if group.histograms.is_empty() {
seen_points(&group, width, op)
seen_points(&group, width)
} else {
// A field's state is one family, so a group that kept histogram payloads exports
// those: a float state beside them would be one column carrying two kinds.
if group.blocks.values().any(|states| {
states
.chunks_exact(words)
.any(|state| state_seen(state, op))
}) {
if group
.blocks
.values()
.any(|states| states.iter().any(|state| state.seen))
{
return Err(DataFusionError::Execution(
"an aggregate field cannot carry both float and histogram state".into(),
));
@@ -224,8 +221,8 @@ impl FloatAggregator {
continue;
}
for (&block, block_states) in &group.blocks {
for (offset, point) in block_states.chunks_exact(point_words).enumerate() {
if !point.chunks_exact(words).any(|state| state_seen(state, op)) {
for (offset, point) in block_states.chunks_exact(width).enumerate() {
if !point.iter().any(|state| state.seen) {
continue;
}
let position = block * BLOCK_POINTS + offset;
@@ -235,8 +232,8 @@ impl FloatAggregator {
)));
}
extend_spans(&mut spans, position);
for (field, state) in point.chunks_exact(words).enumerate() {
states[field].push(state_partial(state, op).map(StatePayload::Float));
for (field, state) in point.iter().enumerate() {
states[field].push(state.partial().map(StatePayload::Float));
}
}
}
@@ -302,14 +299,12 @@ pub(super) fn exported_payload_bytes(accumulator: &HistogramAccumulator) -> usiz
}
/// The points of `group` that carry a state: the ones a partial of it holds.
fn seen_points(group: &Group, width: usize, op: AggregateOp) -> usize {
let words = op.state_words();
let point_words = width.saturating_mul(words);
fn seen_points(group: &Group, width: usize) -> usize {
group.blocks.values().fold(0usize, |count, states| {
count.saturating_add(
states
.chunks_exact(point_words)
.filter(|point| point.chunks_exact(words).any(|state| state_seen(state, op)))
.chunks_exact(width)
.filter(|point| point.iter().any(|state| state.seen))
.count(),
)
})
@@ -686,10 +681,9 @@ impl FloatMerger {
budget: &mut impl FnMut(usize) -> Result<()>,
) -> Result<()> {
let width = self.schema.fields.len();
let op = self.op;
let words = op.state_words();
let point_words = width.saturating_mul(words);
let states_bytes = block_bytes(width, op);
let block_bytes = BLOCK_POINTS
.saturating_mul(width)
.saturating_mul(size_of::<FloatState>());
let blocks = &mut self.groups[index].blocks;
let block_index = point / BLOCK_POINTS;
let entries = blocks.len();
@@ -699,24 +693,21 @@ impl FloatMerger {
budget(
self.retained
.saturating_add(live)
.saturating_add(states_bytes)
.saturating_add(block_bytes)
.saturating_add(btree_backing_bytes(entries.saturating_add(1))),
)?;
self.retained = self
.retained
.saturating_add(states_bytes)
.saturating_add(block_bytes)
.saturating_add(btree_backing_bytes(entries.saturating_add(1)))
.saturating_sub(btree_backing_bytes(entries));
entry.insert(vec![0.0; BLOCK_POINTS * point_words].into_boxed_slice())
entry.insert(vec![FloatState::default(); BLOCK_POINTS * width].into_boxed_slice())
}
};
let offset = (point % BLOCK_POINTS) * point_words;
for (slot, state) in block[offset..offset + point_words]
.chunks_exact_mut(words)
.zip(states)
{
let offset = (point % BLOCK_POINTS) * width;
for (slot, state) in block[offset..offset + width].iter_mut().zip(states) {
if let Some(state) = state {
FloatState::new(slot, op).merge_partial(&state);
slot.merge(self.op, FloatState::from_partial(state));
}
}
Ok(())
@@ -915,166 +906,97 @@ fn state_at(
Ok(Some(state))
}
/// The exported form of the state in `words`, or `None` if it has seen no sample.
///
/// Only the components the op keeps are filled in; the rest stay at their defaults, exactly as the
/// frame that carries them leaves them out.
fn state_partial(words: &[f64], op: AggregateOp) -> Option<AccumulatorState> {
if !state_seen(words, op) {
return None;
impl FloatState {
fn partial(&self) -> Option<AccumulatorState> {
self.seen.then_some(AccumulatorState {
value: self.value,
mean: self.mean,
compensation: self.compensation,
count: self.count,
incremental_mean: self.incremental_mean,
})
}
Some(match op {
// `value`, its compensation, then the presence flag.
AggregateOp::Sum => AccumulatorState {
value: words[0],
compensation: words[1],
..Default::default()
},
// `value`, `compensation`, `mean`, the incremental flag, then the count.
AggregateOp::Avg => AccumulatorState {
value: words[0],
compensation: words[1],
mean: words[2],
incremental_mean: words[3] != 0.0,
count: words[4],
},
AggregateOp::Count => AccumulatorState {
count: words[0],
..Default::default()
},
AggregateOp::Min | AggregateOp::Max => AccumulatorState {
value: words[0],
..Default::default()
},
AggregateOp::Group => AccumulatorState::default(),
// `value`, `mean`, then the count.
AggregateOp::Stddev | AggregateOp::Stdvar => AccumulatorState {
value: words[0],
mean: words[1],
count: words[2],
..Default::default()
},
})
}
impl<'a> FloatState<'a> {
/// Writes `other` into this state, for a state that has seen nothing yet.
fn store_partial(self, other: &AccumulatorState) {
let (op, words) = (self.op, self.words);
match op {
AggregateOp::Count => words[0] = other.count,
// Presence alone, whatever the other side keeps: `group` answers 1 either way.
AggregateOp::Group => words[0] = 1.0,
AggregateOp::Min | AggregateOp::Max => {
words[0] = other.value;
words[1] = 1.0;
}
AggregateOp::Sum => {
words[0] = other.value;
words[1] = other.compensation;
words[2] = 1.0;
}
AggregateOp::Stddev | AggregateOp::Stdvar => {
words[0] = other.value;
words[1] = other.mean;
words[2] = other.count;
}
AggregateOp::Avg => {
words[0] = other.value;
words[1] = other.compensation;
words[2] = other.mean;
words[3] = if other.incremental_mean { 1.0 } else { 0.0 };
words[4] = other.count;
}
fn from_partial(state: AccumulatorState) -> Self {
Self {
seen: true,
value: state.value,
mean: state.mean,
compensation: state.compensation,
count: state.count,
incremental_mean: state.incremental_mean,
}
}
/// Folds the state of another partition into this one.
fn merge_partial(self, other: &AccumulatorState) {
if !state_seen(self.words, self.op) {
self.store_partial(other);
fn merge(&mut self, op: AggregateOp, other: Self) {
if !self.seen {
*self = other;
return;
}
let (op, words) = (self.op, self.words);
match op {
AggregateOp::Sum => {
(words[0], words[1]) =
add_compensated((words[0], words[1]), (other.value, other.compensation));
(self.value, self.compensation) = add_compensated(*self, other);
}
AggregateOp::Count => words[0] += other.count,
AggregateOp::Count => self.count += other.count,
AggregateOp::Group => {}
AggregateOp::Max => {
if words[0] < other.value || words[0].is_nan() {
words[0] = other.value;
if self.value < other.value || self.value.is_nan() {
self.value = other.value;
}
}
AggregateOp::Min => {
if words[0] > other.value || words[0].is_nan() {
words[0] = other.value;
if self.value > other.value || self.value.is_nan() {
self.value = other.value;
}
}
AggregateOp::Avg => merge_avg(words, other),
AggregateOp::Avg => self.merge_avg(other),
AggregateOp::Stddev | AggregateOp::Stdvar => {
// Chan et al.'s pairwise update. `delta * (delta * weight)` keeps the product
// finite where it is: `delta²` alone overflows for `[0]` and `[1.5e154]`, whose
// merged sum of squares is finite.
let count = words[2] + other.count;
let delta = other.mean - words[1];
let weight = words[2] * other.count / count;
words[1] += delta * (other.count / count);
words[0] = words[0] + other.value + delta * (delta * weight);
words[2] = count;
let count = self.count + other.count;
let delta = other.mean - self.mean;
let weight = self.count * other.count / count;
self.mean += delta * (other.count / count);
self.value = self.value + other.value + delta * (delta * weight);
self.count = count;
}
}
}
}
/// Adds the sums while they stay finite, and otherwise continues with the count-weighted mean, as
/// the serial update switches to an incremental mean once its sum overflows.
///
/// The state words are `value`, `compensation`, `mean`, the incremental flag and the count.
fn merge_avg(words: &mut [f64], other: &AccumulatorState) {
let count = words[4] + other.count;
if words[3] == 0.0 && !other.incremental_mean {
let (value, compensation) =
add_compensated((words[0], words[1]), (other.value, other.compensation));
if !value.is_infinite() {
words[0] = value;
words[1] = compensation;
words[4] = count;
return;
/// Adds the sums while they stay finite, and otherwise continues with the count-weighted
/// mean, as the serial update switches to an incremental mean once its sum overflows.
fn merge_avg(&mut self, other: Self) {
let count = self.count + other.count;
if !self.incremental_mean && !other.incremental_mean {
let (value, compensation) = add_compensated(*self, other);
if !value.is_infinite() {
self.value = value;
self.compensation = compensation;
self.count = count;
return;
}
}
let (mean, compensation) = self.mean_form();
let (other_mean, other_compensation) = other.mean_form();
let (weight, other_weight) = (self.count / count, other.count / count);
(self.mean, self.compensation) = kahan_inc(
other_mean * other_weight,
mean * weight,
compensation * weight + other_compensation * other_weight,
);
self.incremental_mean = true;
self.count = count;
}
let (mean, compensation) = mean_form(words[0], words[1], words[2], words[3] != 0.0, words[4]);
let (other_mean, other_compensation) = mean_form(
other.value,
other.compensation,
other.mean,
other.incremental_mean,
other.count,
);
let (weight, other_weight) = (words[4] / count, other.count / count);
(words[2], words[1]) = kahan_inc(
other_mean * other_weight,
mean * weight,
compensation * weight + other_compensation * other_weight,
);
words[3] = 1.0;
words[4] = count;
}
/// The mean and its compensation, converting a sum as the serial update does on switching.
fn mean_form(
value: f64,
compensation: f64,
mean: f64,
incremental: bool,
count: f64,
) -> (f64, f64) {
if incremental {
(mean, compensation)
} else {
(value / count, compensation / count)
/// The mean and its compensation, converting a sum as the serial update does on switching.
fn mean_form(&self) -> (f64, f64) {
if self.incremental_mean {
(self.mean, self.compensation)
} else {
(self.value / self.count, self.compensation / self.count)
}
}
}
@@ -1082,9 +1004,9 @@ fn mean_form(
/// Kahan addends: adding the compensation plainly would drop what the compensation already
/// holds once later partials cancel the sums, as merging `[1e100, 1]`, `[-1e100, 1e84]` and
/// `[-1e84]` would lose the `1`.
fn add_compensated(this: (f64, f64), other: (f64, f64)) -> (f64, f64) {
let (value, compensation) = kahan_inc(other.0, this.0, this.1);
kahan_inc(other.1, value, compensation)
fn add_compensated(this: FloatState, other: FloatState) -> (f64, f64) {
let (value, compensation) = kahan_inc(other.value, this.value, this.compensation);
kahan_inc(other.compensation, value, compensation)
}
/// Merges `count_values` results of several partitions. Each partial already carries its full
+13 -194
View File
@@ -616,96 +616,6 @@ fn incremental_retained_bytes_match_full_recount_for_many_groups() {
}
}
/// A state keeps the components its operation folds plus a presence word, which the count already
/// provides where the state has one. That is the whole width of a state, so a `max` state costs two
/// words where an `avg` state costs five instead of every state costing one fixed struct.
#[test]
fn a_state_is_as_wide_as_the_operation_it_belongs_to() {
for (op, words) in [
(AggregateOp::Count, 1),
(AggregateOp::Group, 1),
(AggregateOp::Min, 2),
(AggregateOp::Max, 2),
(AggregateOp::Sum, 3),
(AggregateOp::Stddev, 3),
(AggregateOp::Stdvar, 3),
(AggregateOp::Avg, 5),
] {
assert_eq!(op.state_words(), words, "{op:?}");
let counted = op
.state_fields()
.iter()
.any(|field| matches!(field, StateField::Count));
assert_eq!(
op.state_words(),
op.state_fields().len() + usize::from(!counted),
"{op:?} keeps its own components and nothing else"
);
}
}
/// What a block keeps is what its operation folds: for every operation the fold's retained figure
/// and the recount of its live blocks agree, and a block holds one state per point per field, each
/// as many words as the operation keeps.
#[test]
fn every_operation_recounts_the_states_it_keeps() {
let schema = schema_with_grid_len(&["value", "other"], 256);
for op in [
AggregateOp::Sum,
AggregateOp::Count,
AggregateOp::Avg,
AggregateOp::Min,
AggregateOp::Max,
AggregateOp::Group,
AggregateOp::Stddev,
AggregateOp::Stdvar,
] {
let mut aggregator =
FloatAggregator::try_new(&schema, op, Grouping::By(&["zone"])).unwrap();
assert_eq!(
aggregator.state_bytes(),
aggregator.recounted_state_bytes(),
"{op:?} initial state"
);
let updates = [
input(
&schema,
metric("m", &[("zone", "a")]),
0,
&[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]],
),
input(
&schema,
metric("m", &[("zone", "a")]),
64,
&[vec![Some(4.0)], vec![Some(5.0)]],
),
input(
&schema,
metric("m", &[("zone", "b")]),
128,
&[vec![Some(6.0)], vec![Some(7.0)]],
),
];
for (index, series) in updates.iter().enumerate() {
aggregator.update(series).unwrap();
assert_eq!(
aggregator.state_bytes(),
aggregator.recounted_state_bytes(),
"{op:?} update {index}"
);
}
// Three blocks of two fields, each state as wide as the operation folds it.
let blocks = &aggregator.groups[0].blocks;
assert_eq!(blocks.len(), 2, "{op:?} blocks the first group wrote");
assert_eq!(
blocks.values().map(|block| block.len()).sum::<usize>(),
blocks.len() * BLOCK_POINTS * schema.fields.len() * op.state_words(),
"{op:?} words per block"
);
}
}
/// The unbudgeted entry point is the budgeted one with a no-op budget, so this compares one path
/// with itself: it is a smoke check that the wrapper still runs, not evidence that the kernels are
/// right. The numbers are checked against Prometheus in `aggregates_match_prometheus` and
@@ -769,7 +679,9 @@ fn adjacent_blocks_are_admitted_and_accounted_separately() {
assert_eq!(aggregator.groups[0].blocks.len(), 2);
// A new block is admitted on the retained state alone, so the last admission settles at
// exactly what the fold keeps and no staging buffer is charged for the samples in it.
let block = block_bytes(schema.fields.len(), AggregateOp::Sum);
let block = BLOCK_POINTS
.saturating_mul(schema.fields.len())
.saturating_mul(std::mem::size_of::<FloatState>());
let (first, second) = (
admissions[admissions.len() - 2],
admissions[admissions.len() - 1],
@@ -798,7 +710,9 @@ fn a_rejected_block_allocation_does_not_insert_a_block() {
// The figure a new block's admission adds: the block's states and its tree node, on top of
// whatever the fold already holds. Located by the value, not by which call reaches it, so
// reordering the earlier admissions cannot make this test pass without testing anything.
let block_states = block_bytes(schema.fields.len(), AggregateOp::Sum);
let block_states = BLOCK_POINTS
.saturating_mul(schema.fields.len())
.saturating_mul(std::mem::size_of::<FloatState>());
let block = block_states.saturating_add(btree_backing_bytes(1));
let mut aggregator =
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap();
@@ -842,7 +756,9 @@ fn a_rejected_later_block_keeps_the_blocks_the_group_already_holds() {
// The group and the tree node are already there, so the block's own states are the whole
// charge of the second block.
let later = input(&schema, metric("m", &[]), 64, &[vec![Some(2.0)]]);
let block_states = block_bytes(schema.fields.len(), AggregateOp::Sum);
let block_states = BLOCK_POINTS
.saturating_mul(schema.fields.len())
.saturating_mul(std::mem::size_of::<FloatState>());
let threshold = retained
.saturating_add(btree_backing_bytes(2).saturating_sub(btree_backing_bytes(1)))
.saturating_add(block_states);
@@ -948,7 +864,9 @@ fn the_fold_releases_the_series_workspace_before_it_admits_a_block() {
let state = aggregator.state_bytes();
assert_eq!(aggregator.recounted_state_bytes(), state);
let block = block_bytes(schema.fields.len(), AggregateOp::Sum);
let block = BLOCK_POINTS
.saturating_mul(schema.fields.len())
.saturating_mul(std::mem::size_of::<FloatState>());
let entry = state
.saturating_sub(block)
.saturating_sub(btree_backing_bytes(1));
@@ -967,105 +885,6 @@ fn the_fold_releases_the_series_workspace_before_it_admits_a_block() {
);
}
/// A chunk admits its workspace and the group it may open in one pool call, releases the
/// workspace at the fold entry, and pays for each block it opens: three calls for a chunk that
/// falls in one block, four for one that opens two.
///
/// The fold used to admit the workspace, then a wider bound of it, then the grouping and its key,
/// then the new group on top of them: six admissions for a chunk that opens a group and one block,
/// seven for one that opens two, on top of the runtime's own resize once the chunk is folded.
#[test]
fn a_chunk_admits_its_workspace_release_and_blocks_in_three_or_four_calls() {
let schema = schema(&["value", "other"]);
let one_block = input(
&schema,
metric("m", &[("zone", "a")]),
0,
&[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]],
);
// Two samples of the first field straddle the edge at 64, so the fold opens a second block.
let two_blocks = input(
&schema,
metric("m", &[("zone", "a")]),
63,
&[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]],
);
let admissions = |series: &Series| {
let mut aggregator =
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone"])).unwrap();
let mut calls = Vec::new();
aggregator
.update_budgeted(series, |bytes| {
calls.push(bytes);
Ok(())
})
.unwrap();
assert_eq!(aggregator.state_bytes(), aggregator.recounted_state_bytes());
calls
};
let one = admissions(&one_block);
assert_eq!(
one.len(),
3,
"the workspace, the release and one block: {one:?}"
);
assert!(
one[0] > one[1],
"the workspace was admitted beside the retained state: {one:?}"
);
let block = block_bytes(schema.fields.len(), AggregateOp::Sum);
assert_eq!(
one[2],
one[1]
.saturating_add(block)
.saturating_add(btree_backing_bytes(1)),
"the block is admitted on the retained state alone"
);
let two = admissions(&two_blocks);
assert_eq!(
two.len(),
4,
"the workspace, the release and two blocks: {two:?}"
);
assert!(two[3] > two[2], "each block is admitted separately");
}
/// A chunk that only writes into states its group already holds costs the workspace admission and
/// the release, and no block.
#[test]
fn a_chunk_of_existing_blocks_admits_only_its_workspace_and_release() {
let schema = schema(&["value"]);
let mut aggregator =
FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone"])).unwrap();
aggregator
.update(&input(
&schema,
metric("m", &[("zone", "a")]),
0,
&[vec![Some(1.0)]],
))
.unwrap();
let mut calls = Vec::new();
aggregator
.update_budgeted(
&input(
&schema,
metric("m", &[("zone", "a")]),
1,
&[vec![Some(2.0)]],
),
|bytes| {
calls.push(bytes);
Ok(())
},
)
.unwrap();
assert_eq!(calls.len(), 2, "the workspace and the release: {calls:?}");
assert_eq!(aggregator.state_bytes(), aggregator.recounted_state_bytes());
}
/// A block a field holds no value in is not allocated: two fields whose samples fall in different
/// blocks leave the group with the blocks they write, not the blocks between them.
#[test]
@@ -2164,7 +1983,7 @@ fn merge_charges_one_block_per_new_block_not_per_point() {
aggregator.update(&series).unwrap();
let partial = aggregator.finish_partial().unwrap();
assert_eq!(partial.groups[0].states[0].len(), 64);
let block = block_bytes(64, AggregateOp::Sum);
let block = BLOCK_POINTS * 64 * std::mem::size_of::<FloatState>();
let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap();
let (peak, result) =
peak_admission(|budget| merge_partial_budgeted(&mut merger, partial, budget));
+2 -2
View File
@@ -44,8 +44,8 @@ pub use empty_metric::{EmptyMetric, EmptyMetricExec, EmptyMetricStream, build_sp
pub use fused_series::is_canonical_producer_exec;
pub use fused_series::{
FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec,
FusedSeriesStream, InputLocality, input_locality, keys_survive_metadata_drop,
matching_operands_drop_key, tree_source_fields,
FusedSeriesStream, InputLocality, input_locality, matching_operands_drop_key,
tree_source_fields,
};
pub use histogram_fold::{
HistogramFold, HistogramFoldExec, HistogramFoldOperation, HistogramFoldStream,
+21 -63
View File
@@ -78,7 +78,6 @@ use datafusion::physical_plan::{
Partitioning, PlanProperties, RecordBatchStream, SendableRecordBatchStream,
};
use datatypes::data_type::DataType as _;
use datatypes::schema::ColumnSchema;
use datatypes::series::SeriesOrdering;
use datatypes::series::frame::{
FrameContent, FrameLabelMode, SeriesFrameDeclaration, aggregate_state_frame_schema,
@@ -97,7 +96,7 @@ use crate::extension_plan::series_repartition::SeriesRepartitionExec;
use crate::extension_plan::{RawSeriesSourceExec, SeriesRouting};
use crate::grid_series::METADATA_LABELS;
use crate::histogram_eval::HISTOGRAM_ROUTING_EXCLUDED_LABELS;
use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor, SourceFacts};
use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor};
use crate::native::metrics::{NodeMetricsSnapshot, is_node_metric};
use crate::native::plan::{
AggregateStage, BoundMap, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree,
@@ -991,7 +990,6 @@ impl FusedSeries {
output_schema,
self.binding.time_index.clone(),
tags,
self.binding.metric_name.clone(),
self.format,
target_partitions,
self.canonical_producer,
@@ -2127,13 +2125,6 @@ pub struct FusedSeriesExec {
time_column: String,
/// The tag column names, indexed by [`OutputColumn::Tag`].
tags: Arc<[String]>,
/// The one metric every input series is named, taken from the binding.
///
/// The canonical pass judged a dropped label set by this: a node that drops the metric name is
/// only told to stream when everything below it is named the same one. The execution reads it
/// so the collection it performs at run time is decided by that same proof; `None` is a binding
/// that names no metric - one raw source per input - and every consumer then collects.
metric_name: Option<Arc<str>>,
/// The columns this node produces.
output_schema: SchemaRef,
/// The evidence claimed by this plan's output partitions.
@@ -2181,42 +2172,20 @@ struct StoredSeriesLabels {
/// Every tag name a stored series reports besides its metric name.
tags: Arc<[Arc<str>]>,
/// Whether the primary key keeps telling two stored series apart once a name-dropping kernel
/// has normalized their labels; see [`keys_survive_metadata_drop`] for the rule itself.
/// has normalized their labels.
///
/// Two facts make that true. The tags and the primary key must be free of the metadata labels:
/// `__type__` and `__unit__` go with the metric name when a kernel drops it, so a binding that
/// keys a series by either of them stops telling two series apart once the name is gone. And
/// every primary-key column must always decode a value, which is what a non-nullable column
/// states. A kernel that drops the name drops every label whose value is empty with it, and a
/// nullable column lets one series hold `NULL` while another holds `''`: two different storage
/// keys, which decode to an absent label and to an empty one, so both are handed over as no
/// label at all once the empty one is dropped. Two such series that agree on every other tag
/// then leave as one identity, whether they arrive in one partition or in two.
keys_survive_metadata_drop: bool,
}
/// Whether the storage key of a scan keeps telling its stored series apart once a name-dropping
/// kernel has normalized their labels.
///
/// `logical_tags` are the tag names every stored series of the scan reports, and `key_columns` are
/// the columns the scan keys one series by. Two facts make the drop injective, and both are read
/// from the scan's own binding rather than from a tree above it:
///
/// - Neither the tags nor the key columns may be a metadata label. `__type__` and `__unit__` go with
/// the metric name when a kernel drops it, so a scan that keys a series by either of them stops
/// telling two series apart once the name is gone.
/// - Every key column must always decode a value, which is what a non-nullable column states. A
/// kernel that drops the name drops every label whose value is empty with it, and a nullable
/// column lets one series hold `NULL` while another holds `''`: two different storage keys, which
/// decode to an absent label and to an empty one, so both are handed over as no label at all once
/// the empty one is dropped. Two such series that agree on every other tag then leave as one
/// identity, whether they arrive in one partition or in two.
///
/// This is the rule the logical canonical pass applies to decide whether a node that drops the
/// metadata labels has to be collected, and the one the physical per-series scan applies to decide
/// whether such a node can stream. Both read it here so the two answers cannot drift apart: the
/// planner reads it from the resolved table's metadata, the execution from the bound scan.
pub fn keys_survive_metadata_drop<'a, 'b>(
logical_tags: impl IntoIterator<Item = &'a str>,
key_columns: impl IntoIterator<Item = &'b ColumnSchema>,
) -> bool {
let metadata = |name: &str| METADATA_LABELS.contains(&name);
logical_tags.into_iter().all(|tag| !metadata(tag))
&& key_columns
.into_iter()
.all(|column| !metadata(&column.name) && !column.is_nullable())
}
impl StoredSeriesLabels {
/// The labels `label_source` proves about every stored series it reads.
fn of(label_source: &RawLabelSource) -> Option<Self> {
@@ -2230,17 +2199,19 @@ impl StoredSeriesLabels {
else {
return None;
};
let metadata = |name: &str| METADATA_LABELS.contains(&name);
Some(Self {
tags: logical_tags
.iter()
.map(|tag| Arc::clone(&tag.name))
.collect(),
keys_survive_metadata_drop: keys_survive_metadata_drop(
logical_tags.iter().map(|tag| tag.name.as_ref()),
physical_metadata
.primary_key_columns()
.map(|column| &column.column_schema),
),
keys_survive_metadata_drop: logical_tags.iter().all(|tag| !metadata(&tag.name))
&& physical_metadata.primary_key_columns().all(|column| {
// A nullable key column can be `NULL` in one series and `''` in another, and a
// name-dropping kernel turns both into no label; only a column that always
// decodes a value keeps the two apart under such a kernel.
!metadata(&column.column_schema.name) && !column.column_schema.is_nullable()
}),
})
}
@@ -2519,7 +2490,6 @@ impl FusedSeriesExec {
output_schema,
time_column,
tags,
None,
FusedOutputFormat::PlainRows,
None,
false,
@@ -2534,7 +2504,6 @@ impl FusedSeriesExec {
output_schema: SchemaRef,
time_column: String,
tags: Vec<String>,
metric_name: Option<Arc<str>>,
format: FusedOutputFormat,
matching_parallelism: Option<usize>,
canonical_producer: bool,
@@ -2818,7 +2787,6 @@ impl FusedSeriesExec {
classification: classification.into(),
time_column,
tags: tags.into(),
metric_name,
output_schema,
duplicate_evidence: Mutex::new(EvidenceClaims {
evidence: None,
@@ -2970,7 +2938,6 @@ impl ExecutionPlan for FusedSeriesExec {
self.output_schema.clone(),
self.time_column.clone(),
self.tags.to_vec(),
self.metric_name.clone(),
self.format,
self.matching_parallelism,
self.canonical_producer,
@@ -3003,19 +2970,10 @@ impl ExecutionPlan for FusedSeriesExec {
evidence: Arc::clone(evidence),
active: true,
});
// The canonical pass decided where a merge belongs from the labels the binding names; the
// execution judges the collection it performs by the same fact, so an already-streaming
// window is not turned into a collecting one here.
let sources = SourceFacts::uniform(
self.tags.to_vec(),
self.metric_name.clone(),
self.series_schemas.len(),
);
let mut state = match NativeExecutionState::try_new_with_sources(
let mut state = match NativeExecutionState::try_new(
&self.tree,
Arc::clone(&context.runtime_env().memory_pool),
evidence.clone(),
&sources,
) {
Ok(state) => state,
Err(error) => {
@@ -238,24 +238,6 @@ fn window_tree_of(function: RangeFunction) -> NativeTree {
NativeTree::try_new(root).expect("a window function keeps the raw input and its grid")
}
/// `rate(x[5m])` as the canonical pass hands it over: one merge over the window's output.
///
/// A range function drops the metric name, so two series that agreed on every other label leave the
/// window under one label set. Collecting them is what refuses a second value at one position: the
/// merge groups by the label set the window handed out and reports the collision, as Prometheus
/// refuses two series of one label set.
fn canonical_rate_tree_of(schema: Arc<SeriesSchema>) -> NativeTree {
let window = rate_tree_of(schema);
let output = window.root().output.clone();
NativeTree::try_new(NativeNode {
expr: NativeExpr::Merge {
input: Box::new(window.root().clone()),
},
output,
})
.expect("a merge over a window on a grid is a legal tree")
}
/// `left + right` on one grid, both operands reading the one actual source.
fn binary_tree(left_schema: Arc<SeriesSchema>, right_schema: Arc<SeriesSchema>) -> NativeTree {
binary_tree_over_sources(BinaryOp::Add, left_schema, right_schema, 0, 0)
@@ -1963,11 +1945,7 @@ fn zone_output_schema() -> SchemaRef {
/// `rate(x[5m])` over frames that carry `zone` as well as `host`, with both in the output.
fn zone_node() -> FusedSeries {
zone_node_with(rate_tree())
}
/// The same node over `tree`, so a test can stage the tree the canonical pass produces.
fn zone_node_with(tree: NativeTree) -> FusedSeries {
let tree = rate_tree();
let binding = binding_of(
&tree,
raw_schema(),
@@ -3419,7 +3397,7 @@ fn rows(batches: &[RecordBatch]) -> Vec<Row> {
const DUPLICATE_LABEL_SET: &str =
"vector cannot contain metrics with the same labelset {host=\"web-1\"}";
/// Two raw series that are already one Prometheus identity are refused by the merge that would
/// Two raw series that are already one Prometheus identity are refused by the runtime that would
/// have to answer both.
///
/// `{host="web-1", zone=""}` and `{host="web-1"}` are one identity to Prometheus: an empty label
@@ -3427,16 +3405,12 @@ const DUPLICATE_LABEL_SET: &str =
/// What the two *raw* keys differ in is the empty `zone` against the absent one, which the scan
/// encodes differently and the decoder therefore opens two input series for. `rate` drops the
/// metric name and the empty value, so the second raw series arrives on the canonical label set the
/// first one already occupies, and the merge collecting the window's output refuses it: the
/// refusal names that label set rather than answering it, because answering would report one point
/// twice.
/// first one already occupies; the refusal names that label set rather than answering it, because
/// answering would report one point twice.
#[tokio::test]
async fn differing_raw_keys_that_close_on_one_label_set_are_refused() {
let exec = plan_of(
zone_node_with(canonical_rate_tree_of(raw_schema())),
memory_exec(frames(&zone_chunks())),
)
.expect("the runtime executes a canonical rate tree");
let exec = plan_of(zone_node(), memory_exec(frames(&zone_chunks())))
.expect("the runtime executes a rate tree");
let error = collect(exec, SessionContext::default().task_ctx())
.await
.expect_err("the second raw key cannot be answered under the first one's label set");
@@ -3450,25 +3424,13 @@ async fn differing_raw_keys_that_close_on_one_label_set_are_refused() {
///
/// The decoder holds the batch it is reading, the chunks it decoded from it and its own source, so a
/// stream that kept its input after the error would hold all three until the consumer drops it. The
/// error staged here is the truncated second series: the first one's rows are handed out, and the
/// second one ends the input without saying it was its last chunk, which the frame decoder refuses.
/// collision above is the error: it happens on the second series, after the first one's rows have
/// been handed out.
#[tokio::test]
async fn a_runtime_error_after_rows_releases_the_input() {
let dropped = Arc::new(AtomicBool::new(false));
let schema = frame_schema();
// The second series is cut off: its chunk does not close it.
let mut chunks = zone_chunks();
chunks.pop().expect("the collision stages two series");
let samples = series_samples(1.0);
chunks.push(chunk_of(
&raw_schema(),
"zone-absent",
metric(&[(HOST, "web-1")], Some(METRIC)),
&times_of(&samples),
vec![values_of(&samples)],
false,
));
let batches = frames(&chunks);
let batches = frames(&zone_chunks());
let source = staged(schema.clone(), {
let dropped = Arc::clone(&dropped);
move || Box::pin(StagedStream::new(schema.clone(), batches.clone()).dropped(&dropped))
@@ -3490,7 +3452,7 @@ async fn a_runtime_error_after_rows_releases_the_input() {
}
};
assert!(
error.to_string().contains("ended before key was final"),
error.to_string().contains(DUPLICATE_LABEL_SET),
"refused for another reason: {error}"
);
// The first series' rows were handed out before the error, and the input is gone although the
@@ -3505,17 +3467,16 @@ async fn a_runtime_error_after_rows_releases_the_input() {
);
}
/// A collision produced inside one native execution is refused by the merge that collects it.
/// A collision produced inside one native execution is still refused by its local runtime state.
///
/// This is the collision the canonical label set exists for, at the layer that sees it:
/// `http_requests_total{host="web-1"}` and `http_errors_total{host="web-1"}` reach the window as two
/// identities and leave it as `{host="web-1"}`. [`FusedSeries`] reads one metric per scan, so this
/// collision cannot be staged through it; the runtime is driven directly, with the names on the raw
/// chunks. The window is handed over collected, as the canonical pass places it: nothing is answered
/// until its input ends, and the merge then refuses the second value of the one label set.
/// chunks.
#[test]
fn two_differently_named_series_collide_once_rate_drops_the_name() {
let tree = canonical_rate_tree_of(raw_schema());
let tree = rate_tree();
let mut execution = NativeExecutor::new(
NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None)
.expect("the runtime executes a rate tree"),
@@ -3534,6 +3495,13 @@ fn two_differently_named_series_collide_once_rate_drops_the_name() {
),
)
.expect("the first series is a legal raw chunk");
assert!(
execution
.next_chunk()
.expect("the first series runs")
.is_some(),
"the first series is answered"
);
execution
.push_raw(
InputId::new(0),
@@ -3547,11 +3515,8 @@ fn two_differently_named_series_collide_once_rate_drops_the_name() {
),
)
.expect("the second series is a legal raw chunk");
// Both series sit in the merge's collection: the second one puts a value where the first one
// already holds one, and the merge refuses the pair as it collects them.
let error = execution
.end_of_input()
.and_then(|()| execution.next_chunk().map(|_| ()))
.next_chunk()
.expect_err("the second series puts a value where the first one already holds one");
assert!(
error.to_string().contains(DUPLICATE_LABEL_SET),
@@ -94,8 +94,6 @@
//! until its per-input close/EOF barrier is satisfied, then
//! reserve merge workspace for that identity alone. This bounds merge workspace to one pending
//! identity, while its inputs remain charged until handoff. No task waits for memory admission.
//! An output registers one consumer with the pool and splits every identity's charge off it, so the
//! pool holds an entry per output stream rather than one per series that stream routes.
//!
//! An output ends only when every input partition finished and its queue was drained; a chain that
//! closed locally is not the end of anything. Each input sends its own EOF to each live output before
@@ -2109,14 +2107,6 @@ struct RepartitionStream {
inputs: usize,
output: VecDeque<ReadySeries>,
handoff: Arc<Mutex<Handoff>>,
/// What every identity of this output is charged to.
///
/// One registration for the whole output stream: each identity's descriptor, each owner's
/// merge workspace and each raw owner's reconstruction are reservations split off it, so the
/// pool holds an entry per output partition instead of one per series the output routes. A
/// reservation split off it still keeps its bytes live until it is dropped, and the pool sees
/// each of those bytes exactly once.
groups_reservation: MemoryReservation,
/// Whether a new pass over the pending identities is owed after an input EOF.
///
/// A chain can only complete the identity it carries, so the chain arm checks that group
@@ -2200,7 +2190,6 @@ impl RepartitionStream {
inputs,
output: VecDeque::new(),
handoff: Arc::new(Mutex::new(Handoff::default())),
groups_reservation: MemoryConsumer::new("SeriesRepartitionGroups").register(pool),
rescan: false,
resume_after: None,
reported: false,
@@ -2218,7 +2207,7 @@ impl RepartitionStream {
let rows = group.fragments.iter().fold(0usize, |rows, series| {
rows.saturating_add(series.time.len())
});
let reservation = self.groups_reservation.new_empty();
let reservation = MemoryConsumer::new("SeriesRepartitionOwner").register(&self.pool);
reservation.try_grow(merge_workspace_bytes(
rows,
group.fragments.len(),
@@ -2292,7 +2281,7 @@ impl RepartitionStream {
// Reconstruction's working set depends on how the pieces interleave, which is only known
// while it runs: charge each allocation as it is made rather than reserving a worst case
// that a well-ordered identity never approaches.
let workspace = self.groups_reservation.new_empty();
let workspace = MemoryConsumer::new("SeriesRepartitionRawOwner").register(&self.pool);
let mut reserve = |bytes: usize| workspace.try_grow(bytes);
let mut shrink = |bytes: usize| {
let result = workspace.try_shrink(bytes);
@@ -2491,7 +2480,8 @@ impl Stream for RepartitionStream {
}
};
if !this.groups.contains_key(&key) {
let metadata = this.groups_reservation.new_empty();
let metadata =
MemoryConsumer::new("SeriesRepartitionIdentity").register(&this.pool);
let key_bytes =
match &key {
OwnerKey::Evaluated((name, tags)) => {
@@ -521,77 +521,13 @@ impl ExecutionPlan for Scripted {
/// A task context whose memory pool is `pool`, so what the node charges can be read afterwards.
fn task_context(pool: &Arc<GreedyMemoryPool>) -> Arc<TaskContext> {
context_over(Arc::clone(pool) as Arc<dyn MemoryPool>)
}
/// A task context over `pool`, for a test that keeps its own pool to read it afterwards.
fn context_over(pool: Arc<dyn MemoryPool>) -> Arc<TaskContext> {
let runtime = RuntimeEnvBuilder::new()
.with_memory_pool(pool)
.with_memory_pool(Arc::clone(pool) as Arc<dyn MemoryPool>)
.build()
.expect("the runtime environment builds");
Arc::new(TaskContext::default().with_runtime(Arc::new(runtime)))
}
/// A pool that counts the consumers registered with it, so a test can see how many entries the node
/// adds whatever the input carries.
#[derive(Debug)]
struct RegistrationCountingPool {
inner: GreedyMemoryPool,
registrations: AtomicUsize,
}
impl RegistrationCountingPool {
fn new(bytes: usize) -> Arc<Self> {
Arc::new(Self {
inner: GreedyMemoryPool::new(bytes),
registrations: AtomicUsize::new(0),
})
}
/// How many consumers were registered with the pool, in total.
fn registrations(&self) -> usize {
self.registrations.load(Ordering::SeqCst)
}
}
impl fmt::Display for RegistrationCountingPool {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(f, "registration counting test pool")
}
}
impl MemoryPool for RegistrationCountingPool {
fn name(&self) -> &str {
"registration counting test pool"
}
fn register(&self, consumer: &MemoryConsumer) {
self.registrations.fetch_add(1, Ordering::SeqCst);
self.inner.register(consumer);
}
fn unregister(&self, consumer: &MemoryConsumer) {
self.inner.unregister(consumer);
}
fn grow(&self, reservation: &MemoryReservation, additional: usize) {
self.inner.grow(reservation, additional);
}
fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
self.inner.shrink(reservation, shrink);
}
fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> DataFusionResult<()> {
self.inner.try_grow(reservation, additional)
}
fn reserved(&self) -> usize {
self.inner.reserved()
}
}
/// The node over `child` routing into `outputs` partitions.
fn node(child: Arc<dyn ExecutionPlan>, outputs: usize) -> Arc<SeriesRepartitionExec> {
Arc::new(SeriesRepartitionExec::try_new(child, outputs).expect("the node is built"))
@@ -2756,42 +2692,6 @@ async fn single_fragment_large_identity_fits_finite_payload_budget() {
);
}
/// Each identity used to register a pool consumer of its own, so the pool's bookkeeping grew with
/// the number of series an output routed. The stream registers once per output stream now, and the
/// identity's descriptor, an owner's merge workspace and a raw owner's reconstruction are
/// reservations split off it: more identities must not mean more entries, and the bytes each of
/// them holds are still released.
#[tokio::test]
async fn pool_entries_follow_output_streams_not_identities() {
let registrations = |identities: usize| async move {
let steps: Vec<Step> = (0..identities)
.map(|index| Step::chunk(chunk(labels(&format!("h{index:03}")), 1, 1, true)))
.collect();
let pool = RegistrationCountingPool::new(1 << 20);
let context = context_over(Arc::clone(&pool) as Arc<dyn MemoryPool>);
let plan: Arc<dyn ExecutionPlan> = node(Scripted::plan(vec![steps]), 1);
let outputs = drain(&plan, &context).await;
let chunks = outputs
.into_iter()
.flat_map(|output| output.expect("the output routes every identity"))
.collect::<Vec<_>>();
assert_eq!(identities, chunks.len(), "each identity is published once");
assert_eq!(0, pool.reserved(), "every charge is released with the read");
pool.registrations()
};
let few = registrations(1).await;
let many = registrations(16).await;
assert_eq!(
few, many,
"the entries the node registers follow the identities: {few} for one and {many} for sixteen"
);
assert!(
few <= 4,
"{few} pool entries for one input and one output stream"
);
}
/// An input EOF already consumed still counts when a different input first contributes an identity.
#[tokio::test]
async fn eof_before_first_identity_contribution_makes_group_ready() {
@@ -4288,83 +4188,6 @@ async fn owner_rejects_conflicting_values_at_same_identity_and_position() {
);
}
/// The owner merges the histogram pieces of one label set that its inputs hold.
///
/// A canonical boundary hands one label set to a single owner, and that owner is where the pieces the
/// producer's partitions collected meet: here two inputs hold complementary positions of one label
/// set and each piece carries its own histogram payload, so the merge rebuilds one field over the
/// union of the positions with every row still naming the payload it came from. The pieces are closed
/// as they arrive, so the merged series is also what proves the closing chunk of a chain survives the
/// merge.
#[tokio::test]
async fn owner_merges_histogram_pieces_of_one_label_set_across_inputs() {
let metric = labels("histogram-pieces");
let (first, closing) = histogram_rows_chunk_with_sum(metric.clone(), 0, 1, 10.0);
let (second, _) = histogram_rows_chunk_with_sum(metric.clone(), 8, 1, 20.0);
let child = Scripted::plan(vec![
vec![Step::Chunk(first, closing)],
vec![Step::chunk((second, true))],
]);
let plan: Arc<dyn ExecutionPlan> = node(child, 2);
let outputs = drain(&plan, &SessionContext::default().task_ctx()).await;
let owner = Routing::new(2).output(&metric);
let chunks = outputs[owner].as_ref().expect("the owner output decodes");
assert_output_contract(chunks, &[(metric.clone(), vec![0, 8])]);
assert_eq!(1, chunks.iter().filter(|(_, closed)| *closed).count());
// One row per valued position, in order, each holding the payload of the piece that pushed it: a
// merge that rebuilt the field without reindexing the payloads would answer one payload twice.
let merged = chunks
.iter()
.flat_map(|(series, _)| {
let field = series.fields.get(0).expect("one field");
series.time.positions().map(move |(position, row)| {
let datatypes::series::samples::SampleRef::Histogram(view) = field.sample(row)
else {
panic!("position {position} of the merged series is not a histogram");
};
(position, view.sum, view.count)
})
})
.collect::<Vec<_>>();
assert_eq!(
vec![(0, 10.0, 4.0), (8, 20.0, 4.0)],
merged,
"the merged histograms are not the ones the pieces carried"
);
}
/// Two histogram pieces of one label set at one position fail the merge.
///
/// The pieces were routed to one owner because they are one label set, so a position both of them
/// fill would carry two values of one field. The kernel reports that the way it does for floats, and
/// the histogram payload makes no difference to it.
#[tokio::test]
async fn owner_rejects_two_histogram_pieces_at_one_position() {
let metric = labels("histogram-conflict");
let (first, _) = histogram_rows_chunk_with_sum(metric.clone(), 4, 1, 10.0);
let (second, _) = histogram_rows_chunk_with_sum(metric.clone(), 4, 1, 20.0);
let child = Scripted::plan(vec![
vec![Step::chunk((first, true))],
vec![Step::chunk((second, true))],
]);
let plan: Arc<dyn ExecutionPlan> = node(child, 1);
let outputs = drain(&plan, &SessionContext::default().task_ctx()).await;
let error = outputs[0]
.as_ref()
.err()
.unwrap_or_else(|| {
panic!(
"two histogram pieces of one label set at one position were merged: {:?}",
outputs[0].as_ref().map(|chunks| summaries(chunks))
)
})
.to_string();
assert!(
error.contains("cannot contain metrics with the same labelset"),
"the failure does not name the shared label set: {error}"
);
}
/// Same-identity input chunks are collected and emitted as one closed series.
#[tokio::test]
async fn same_canonical_key_chains_merge_into_one_series() {
+7 -45
View File
@@ -233,7 +233,13 @@ impl GridSeriesBuilder {
for (field, value) in self.fields.iter_mut().zip(values) {
match value {
Some(GridSample::Histogram(h)) => {
let column = field.start_histogram();
field.values.push(0.0);
field.nulls.append(false);
let rows = field.values.len() - 1;
let column = field.histograms.get_or_insert_with(|| HistogramColumn {
batch: HistogramBatchBuilder::new(),
indices: vec![None; rows],
});
column.indices.push(Some(h.push_to(&mut column.batch)?));
}
Some(GridSample::Float(value)) => field.push_float(Some(*value)),
@@ -243,38 +249,6 @@ impl GridSeriesBuilder {
Ok(())
}
/// Like [`GridSeriesBuilder::push_samples`], with the values taken as the fields hand them out.
///
/// A histogram value goes into its column straight from the view it was read as: the owned
/// form beside this builder exists for the paths that compute a histogram, while a selection
/// that only hands a sample on has nothing to compute and copies nothing. The row this writes
/// and the row [`GridSeriesBuilder::push_samples`] writes are the same, so a value pushed
/// either way leaves the same payload.
pub(crate) fn push_refs(&mut self, index: usize, values: &[SampleRef<'_>]) -> Result<()> {
if !self.start_row(
index,
values.len(),
values
.iter()
.all(|value| matches!(value, SampleRef::Missing)),
)? {
return Ok(());
}
for (field, value) in self.fields.iter_mut().zip(values) {
match value {
SampleRef::Histogram(view) => {
let column = field.start_histogram();
column
.indices
.push(Some(column.batch.push(*view).map_err(external)?));
}
SampleRef::Float(value) => field.push_float(Some(*value)),
SampleRef::Missing => field.push_float(None),
}
}
Ok(())
}
/// Validates the row and retains it; returns whether it holds a value to write.
fn start_row(&mut self, index: usize, len: usize, empty: bool) -> Result<bool> {
if len != self.schema.fields.len() {
@@ -430,18 +404,6 @@ impl FieldBuilder {
column.indices.push(None);
}
}
/// Writes the value columns of a row holding a histogram, opening the histogram column if this
/// is its first value, and hands back the column the new row's index still goes into.
fn start_histogram(&mut self) -> &mut HistogramColumn {
self.values.push(0.0);
self.nulls.append(false);
let rows = self.values.len() - 1;
self.histograms.get_or_insert_with(|| HistogramColumn {
batch: HistogramBatchBuilder::new(),
indices: vec![None; rows],
})
}
}
/// Prometheus `schema.IsMetadataLabel`: the labels removed with the metric name.
+1 -246
View File
@@ -16,9 +16,7 @@ use std::mem::size_of;
use std::sync::Arc;
use datafusion::arrow::array::{Float64Array, UInt32Array};
use datatypes::series::histogram::{
CounterResetHint, HistogramBatch, HistogramBatchBuilder, HistogramView,
};
use datatypes::series::histogram::{HistogramBatch, HistogramBatchBuilder};
use datatypes::series::samples::{
DenseSamplesRef, FieldSpec, HistogramSamples, SampleRef, Samples, Series, SeriesFields,
SeriesSchema,
@@ -301,130 +299,6 @@ fn merge_keeps_histograms_and_rejects_any_doubled_field() {
}
}
/// The histogram payloads of one label set merge by position, whatever table each shard filed its
/// own payload in.
///
/// A label set reaches a merge as one fragment per producer, and every fragment was built by its own
/// builder: each holds its own histogram table and its own indices into it, so the merged field has
/// to be rebuilt over the union of the positions with each row still naming the payload it was
/// pushed with. The fragments here are complementary, the position neither of two fragments holds
/// stays a hole, and a fragment that pushes nothing adds no row.
#[test]
fn merge_keeps_each_shards_histogram_payload() {
// One histogram per shard, each with its own count, sum and bucket payload: `count/sum` are the
// two fields a reader of the merged series checks a payload by.
let payloads = [
OwnedHistogram::parse(
"0/0/0/3ff0000000000000/3ff0000000000000/1/0_2/4000000000000000,4008000000000000///",
),
OwnedHistogram::parse("0/0/0/4008000000000000/4014000000000000/1/1_1/4010000000000000///"),
OwnedHistogram::parse("0/0/0/4014000000000000/4018000000000000/1/0_1/4014000000000000///"),
];
// The `(sum, count)` each payload was written with, in payload order.
let scalars = [(1.0, 1.0), (5.0, 3.0), (6.0, 5.0)];
let schema = grid_schema(1, 8);
let shard = |rows: &[(usize, usize)]| {
let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap();
for &(point, payload) in rows {
let histogram = FloatHistogram::try_from_view(&payloads[payload].view()).unwrap();
builder
.push_samples(point, &[Some(GridSample::Histogram(Box::new(histogram)))])
.unwrap();
}
builder
.finish(metric(0))
.unwrap()
.expect("the shard holds the rows it pushed")
};
// A shard of the same label set holding nothing finishes with no series at all, so it adds
// neither a row nor a gap to what the others pushed.
assert!(
GridSeriesBuilder::try_new(schema.clone())
.unwrap()
.finish(metric(0))
.unwrap()
.is_none(),
"an empty shard handed out a series"
);
let merged = merge_same_labelset(
&schema,
vec![shard(&[(0, 0), (3, 1)]), shard(&[(1, 1)]), shard(&[(4, 2)])],
)
.unwrap();
assert_eq!(merged.len(), 1, "one label set, so one merged series");
let field = merged[0].fields.get(0).expect("one field");
let merged_rows = merged[0]
.time
.positions()
.map(|(point, row)| {
let SampleRef::Histogram(view) = field.sample(row) else {
panic!("position {point} of the merged series is not a histogram");
};
let histogram = FloatHistogram::try_from_view(&view).unwrap();
(point, histogram.sum, histogram.count)
})
.collect::<Vec<_>>();
// One row per valued position, in order: position 2 is a hole in every shard and stays a hole,
// and each row carries the payload its own shard pushed.
assert_eq!(
vec![
(0, scalars[0].0, scalars[0].1),
(1, scalars[1].0, scalars[1].1),
(3, scalars[1].0, scalars[1].1),
(4, scalars[2].0, scalars[2].1),
],
merged_rows
);
// The merged field is one freshly built table over those rows: the rows the shards held are
// addressed by the merged series' own indices, so a row cannot name a shard's table.
let SampleRef::Histogram(first) = field.sample(0) else {
panic!("the merged field holds no histogram");
};
assert_eq!(&[2.0, 3.0], first.positive.buckets);
let SampleRef::Histogram(second) = field.sample(1) else {
panic!("the merged field holds no histogram");
};
assert_eq!(&[4.0], second.positive.buckets);
assert_eq!(1, second.positive.span_lengths.len());
assert_eq!(1, second.positive.span_offsets[0]);
}
/// Two shards of one label set holding a histogram at one position are refused.
///
/// The merge collects the pieces of one label set into one series, so a position two pieces fill
/// would carry two values of one field: that is the collision Prometheus reports rather than a
/// choice for this node to make. The refusal names the shared label set, as it does for floats.
#[test]
fn merge_rejects_a_histogram_position_two_shards_hold() {
let payloads = [
OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/1/0_1/4000000000000000///"),
OwnedHistogram::parse("0/0/0/4000000000000000/4000000000000000/1/0_1/4000000000000000///"),
];
let schema = grid_schema(1, 8);
let shard = |point: usize, payload: usize| {
let histogram = FloatHistogram::try_from_view(&payloads[payload].view()).unwrap();
let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap();
builder
.push_samples(point, &[Some(GridSample::Histogram(Box::new(histogram)))])
.unwrap();
builder.finish(metric(0)).unwrap().unwrap()
};
// Complementary positions merge whatever payloads they hold, so the only reason the second pair
// is refused is the position both of them fill.
let merged = merge_same_labelset(&schema, vec![shard(0, 0), shard(1, 1)]).unwrap();
assert_eq!(1, merged.len());
assert_eq!(
2,
merged[0].time.len(),
"two shards holding different positions merge"
);
let error = merge_same_labelset(&schema, vec![shard(1, 0), shard(1, 1)]).unwrap_err();
assert!(
error.to_string().contains("same labelset"),
"the refusal does not name the shared label set: {error}"
);
}
/// Two values a million positions apart are one series with two rows, addressed by two runs: the
/// positions between them cost neither a row nor a value buffer sized by the span.
#[test]
@@ -1587,122 +1461,3 @@ mod acceptance {
assert_eq!(read(&three_chunks), read(&two_chunks));
}
}
/// The payload of one histogram, field by field, with every float compared by its bits so that a
/// value that moved by one bit is not equal.
fn payload(view: &HistogramView<'_>) -> Vec<u64> {
let hint = match view.reset_hint {
CounterResetHint::Unknown => 0,
CounterResetHint::CounterReset => 1,
CounterResetHint::NotCounterReset => 2,
CounterResetHint::Gauge => 3,
CounterResetHint::Unrecognized(value) => (value as u64) | (1 << 32),
};
let mut bits = vec![
view.schema as u64,
hint,
view.start_timestamp.map_or(u64::MAX, |start| start as u64),
view.zero_threshold.to_bits(),
view.zero_count.to_bits(),
view.count.to_bits(),
view.sum.to_bits(),
view.custom_values.len() as u64,
];
bits.extend(view.custom_values.iter().map(|value| value.to_bits()));
for side in [&view.positive, &view.negative] {
bits.push(side.span_offsets.len() as u64);
bits.push(side.span_lengths.len() as u64);
bits.push(side.buckets.len() as u64);
bits.extend(side.span_offsets.iter().map(|&offset| offset as u64));
bits.extend(side.span_lengths.iter().map(|&length| length as u64));
bits.extend(side.buckets.iter().map(|bucket| bucket.to_bits()));
}
bits
}
/// A value pushed from the view it was read as lands in the result as the payload the owned route
/// writes, start timestamp included: a selection that only hands a sample on leaves it as it is.
#[test]
fn a_pushed_view_writes_the_payload_of_the_owned_route() {
let schema = grid_schema(2, 3);
let histograms = [
OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///"),
OwnedHistogram::parse("0/0/0/4000000000000000/4008000000000000/1/0_1/4000000000000000///"),
];
// The sample a scan read: the payload of its row plus the start timestamp the row carried.
let mut stamped = histograms[1].view();
stamped.start_timestamp = Some(1_500);
let views = [histograms[0].view(), stamped];
let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap();
builder
.push_samples(
0,
&[
Some(GridSample::Histogram(Box::new(
FloatHistogram::try_from_view(&views[0]).unwrap(),
))),
Some(GridSample::Float(2.0)),
],
)
.unwrap();
builder
.push_samples(
2,
&[
None,
Some(GridSample::Histogram(Box::new(
FloatHistogram::try_from_view(&views[1]).unwrap(),
))),
],
)
.unwrap();
let owned = builder.finish(metric(0)).unwrap().unwrap();
let mut builder = GridSeriesBuilder::try_new(schema).unwrap();
builder
.push_refs(0, &[SampleRef::Histogram(views[0]), SampleRef::Float(2.0)])
.unwrap();
builder
.push_refs(2, &[SampleRef::Missing, SampleRef::Histogram(views[1])])
.unwrap();
let by_view = builder.finish(metric(0)).unwrap().unwrap();
let described = |series: &Series, field: usize| -> Vec<(usize, String, Option<Vec<u64>>)> {
let samples = &series.fields.as_slice()[field];
series
.time
.positions()
.map(|(point, row)| match samples.sample(row) {
SampleRef::Missing => (point, "missing".to_string(), None),
SampleRef::Float(value) => {
(point, "float".to_string(), Some(vec![value.to_bits()]))
}
SampleRef::Histogram(view) => {
(point, "histogram".to_string(), Some(payload(&view)))
}
})
.collect()
};
for field in 0..2 {
assert_eq!(
described(&owned, field),
described(&by_view, field),
"field {field} came back different through the view route"
);
}
assert_eq!(
described(&by_view, 1),
vec![
(0, "float".to_string(), Some(vec![2.0f64.to_bits()])),
(2, "histogram".to_string(), Some(payload(&views[1]))),
],
"the pushed view lost the payload it was read with"
);
// The comparison covers the start timestamp, which the two routes had to preserve.
assert_ne!(
payload(&views[1]),
payload(&histograms[1].view()),
"the start timestamp is part of what is compared"
);
}
+7 -17
View File
@@ -15,7 +15,6 @@
//! `sum` and `avg` of native histograms within one aggregation group, following the histogram
//! paths of Prometheus `aggregation`.
use std::borrow::Cow;
use std::sync::Arc;
use datafusion::arrow::array::UInt32Array;
@@ -140,7 +139,7 @@ fn aggregate_point<'a>(
has_float = true;
None
}
SampleRef::Histogram(view) => Some(FloatHistogram::try_from_view(&view).map(Cow::Owned)),
SampleRef::Histogram(view) => Some(FloatHistogram::try_from_view(&view)),
});
let (value, _) = sum_or_avg(op, histograms.by_ref())?;
// A fold stopped by mixed bucket kinds still reports conversion errors of later samples.
@@ -160,17 +159,12 @@ fn aggregate_point<'a>(
/// time, so the first error ends the fold. The result is derived, so it carries no start
/// timestamp.
///
/// The histograms are carried by reference: the accumulator is the only value the fold copies
/// out, and every sample after the first is only read. A window of `w` samples therefore costs
/// one deep copy, not `w`, and a caller that already owns its window pays none for the samples it
/// keeps.
///
/// The flag reports whether any add had to reconcile two different custom-bucket sets, which is
/// what Prometheus annotates with an info. It is carried out of the fold rather than recomputed:
/// the fold already calls [`FloatHistogram::kahan_add`] and receives that outcome.
pub(crate) fn sum_or_avg<'a>(
pub(crate) fn sum_or_avg(
op: HistogramAggregate,
histograms: impl IntoIterator<Item = Result<Cow<'a, FloatHistogram>>>,
histograms: impl IntoIterator<Item = Result<FloatHistogram>>,
) -> Result<(Option<FloatHistogram>, bool)> {
let mut value: Option<FloatHistogram> = None;
let mut compensation: Option<FloatHistogram> = None;
@@ -181,25 +175,23 @@ pub(crate) fn sum_or_avg<'a>(
let h = h?;
count += 1.0;
let Some(current) = value.as_mut() else {
// The one copy the fold takes: the accumulator starts as the first sample.
value = Some(h.into_owned());
value = Some(h);
continue;
};
let h = h.as_ref();
// Go drops the result with a warning for this addition error only; others propagate.
if current.uses_custom_buckets() != h.uses_custom_buckets() {
return Ok((None, custom_reconciled));
}
match op {
HistogramAggregate::Sum => {
let (c, outcome) = current.kahan_add(h, compensation.take())?;
let (c, outcome) = current.kahan_add(&h, compensation.take())?;
custom_reconciled |= outcome.custom_bounds_reconciled;
compensation = Some(c);
}
HistogramAggregate::Avg => {
if mean.is_none() {
let mut sum = current.clone();
let (c, outcome) = sum.kahan_add(h, compensation.clone())?;
let (c, outcome) = sum.kahan_add(&h, compensation.clone())?;
custom_reconciled |= outcome.custom_bounds_reconciled;
if !sum.has_overflow() {
*current = sum;
@@ -218,9 +210,7 @@ pub(crate) fn sum_or_avg<'a>(
if let Some(c) = compensation.as_mut() {
c.mul(q);
}
// The incremental mean scales the sample it adds, so this is the one branch that
// needs the histogram by value on the sample after the first.
let mut to_add = h.clone();
let mut to_add = h;
to_add.div(count);
let mean = mean.as_mut().expect("set above");
mean.mul(q);
@@ -31,46 +31,8 @@ use crate::histogram_eval::native::{CUSTOM_BUCKETS_SCHEMA, bound};
type Buckets = Vec<(i32, f64)>;
/// How often a histogram of this thread was copied or built from a view, so a test can hold a path
/// to the copies it should not make rather than to the shape of its arguments. Test-only: a copy
/// and a borrow produce the same value, so a test comparing only values could not tell the two
/// apart. Both counters belong to the calling thread, so tests running side by side cannot see
/// each other's work.
#[cfg(test)]
pub(crate) mod copies {
use std::cell::Cell;
thread_local! {
/// Deep copies [`super::FloatHistogram::clone`] made on this thread.
static CLONES: Cell<usize> = const { Cell::new(0) };
/// Owned histograms [`super::FloatHistogram::try_from_view`] built on this thread.
static MATERIALIZED: Cell<usize> = const { Cell::new(0) };
}
/// Records one deep copy.
pub(crate) fn record_clone() {
CLONES.with(|clones| clones.set(clones.get() + 1));
}
/// Records one owned histogram built from a view.
pub(crate) fn record_materialized() {
MATERIALIZED.with(|built| built.set(built.get() + 1));
}
/// The deep copies made on this thread since the last call, which starts a new count.
pub(crate) fn take_clones() -> usize {
CLONES.with(|clones| clones.replace(0))
}
/// The owned histograms built from a view on this thread since the last call, which starts a
/// new count.
pub(crate) fn take_materialized() -> usize {
MATERIALIZED.with(|built| built.replace(0))
}
}
/// A native histogram with absolute bucket counts.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub(crate) struct FloatHistogram {
pub(crate) reset_hint: CounterResetHint,
/// The start timestamp of the sample this histogram was read from, in milliseconds.
@@ -112,30 +74,8 @@ pub(crate) struct AddOutcome {
pub(crate) custom_bounds_reconciled: bool,
}
impl Clone for FloatHistogram {
fn clone(&self) -> Self {
#[cfg(test)]
copies::record_clone();
Self {
reset_hint: self.reset_hint,
start_timestamp: self.start_timestamp,
schema: self.schema,
zero_threshold: self.zero_threshold,
zero_count: self.zero_count,
count: self.count,
sum: self.sum,
positive: self.positive.clone(),
negative: self.negative.clone(),
custom_values: self.custom_values.clone(),
layout: self.layout.clone(),
}
}
}
impl FloatHistogram {
pub(crate) fn try_from_view(view: &HistogramView<'_>) -> Result<Self> {
#[cfg(test)]
copies::record_materialized();
Ok(Self {
reset_hint: view.reset_hint,
start_timestamp: view.start_timestamp,
+3 -83
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::sync::Arc;
@@ -23,17 +22,16 @@ use datatypes::series::histogram::{
use datatypes::series::samples::{FieldSpec, SampleRef, Series, SeriesSchema};
use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis};
use super::float_histogram::copies;
use super::float_histogram::{AddOutcome, FloatHistogram};
use super::{
HistogramAggregate, HistogramFunction, HistogramParameters, NativeHistogramFunction,
aggregate_histograms, evaluate_histograms, evaluate_histograms_with_annotations,
histogram_quantiles, histogram_string, map_native_histogram, sum_or_avg,
histogram_quantiles, histogram_string, map_native_histogram,
};
use crate::grid_series::merge_same_labelset;
use crate::grid_series::reference::{
GoldenCase, Labels, OwnedHistogram, Points, assert_histogram_close, canonical_bits, close,
golden_cases, labels_of, metric_of, mixed_series, points_by_labels, points_of,
GoldenCase, Labels, OwnedHistogram, Points, canonical_bits, close, golden_cases, labels_of,
metric_of, mixed_series, points_by_labels, points_of,
};
fn schema(case: &GoldenCase) -> Arc<SeriesSchema> {
@@ -1067,81 +1065,3 @@ fn classic_histograms_keep_metadata_identity_until_after_evaluation() {
);
}
}
/// A window of native histograms is folded by reference: the sum copies the accumulator it starts
/// from and nothing else, so four samples cost one deep copy rather than four. The fold itself is
/// the one the owned samples go through, bit for bit.
#[test]
fn a_histogram_sum_copies_one_sample_of_a_window() {
let window: Vec<FloatHistogram> = [
"0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///",
"0/0/0/4000000000000000/4008000000000000/0/1_1/4000000000000000///",
"0/0/0/4008000000000000/4010000000000000/0/1_1/4008000000000000///",
"0/0/0/4010000000000000/4014000000000000/0/1_1/4010000000000000///",
]
.iter()
.map(|text| FloatHistogram::try_from_view(&OwnedHistogram::parse(text).view()).unwrap())
.collect();
copies::take_clones();
let (sum, reconciled) = sum_or_avg(
HistogramAggregate::Sum,
window.iter().map(|h| Ok(Cow::Borrowed(h))),
)
.unwrap();
assert_eq!(
copies::take_clones(),
1,
"the sum copied every sample of the window, not only the accumulator it starts from"
);
assert!(!reconciled);
// The same window as owned samples, taken before the count: the accumulator arrives by value,
// so the fold copies nothing and must produce the result the borrowed fold produced.
let owned_samples: Vec<Cow<'_, FloatHistogram>> =
window.iter().cloned().map(Cow::Owned).collect();
copies::take_clones();
let (owned, _) =
sum_or_avg(HistogramAggregate::Sum, owned_samples.into_iter().map(Ok)).unwrap();
assert_eq!(copies::take_clones(), 0, "an owned sample was copied");
let sum = sum.expect("the window sums to a histogram");
assert_histogram_close(
&sum,
&owned.expect("the window sums to a histogram"),
0.0,
"the borrowed sum",
);
}
/// The mean reads the window by reference too, and takes the same samples by value as the owned
/// fold does: the sum `kahan_add` keeps is copied either way, and the values agree bit for bit.
#[test]
fn a_histogram_mean_reads_its_window_by_reference() {
let window: Vec<FloatHistogram> = [
"0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///",
"0/0/0/4000000000000000/4008000000000000/0/1_1/4000000000000000///",
"0/0/0/4008000000000000/4010000000000000/0/1_1/4008000000000000///",
]
.iter()
.map(|text| FloatHistogram::try_from_view(&OwnedHistogram::parse(text).view()).unwrap())
.collect();
let borrowed = sum_or_avg(
HistogramAggregate::Avg,
window.iter().map(|h| Ok(Cow::Borrowed(h))),
)
.unwrap();
let owned = sum_or_avg(
HistogramAggregate::Avg,
window.iter().cloned().map(|h| Ok(Cow::Owned(h))),
)
.unwrap();
assert_eq!(borrowed.1, owned.1);
assert_histogram_close(
&borrowed.0.expect("the window averages to a histogram"),
&owned.0.expect("the window averages to a histogram"),
0.0,
"the borrowed mean",
);
}
+2 -22
View File
@@ -27,22 +27,6 @@ use crate::map::drop_metric_name;
#[cfg(test)]
mod tests;
/// The source rows [`select_instants`] examined, counted per row.
///
/// The selector advances one cursor over the axis, so the rows it reads can never outnumber the
/// rows of the axis: one pass, not one per grid point. Counting them is what keeps a second pass
/// over the axis from being added back unobserved, since the selection it returns is the same
/// either way.
#[cfg(test)]
pub(crate) static AXIS_ROWS_READ: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
/// Counts one row of the axis examined; a no-op outside tests.
fn row_read() {
#[cfg(test)]
AXIS_ROWS_READ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
/// Selects the one source row an anchored instant reads, with the lookback, offset and staleness
/// rules of [`select_instants`]: the last sample at or before `at_ms` (shifted by `offset_ms`) and
/// not more than the lookback before it, unless a stale marker is the latest sample there.
@@ -117,13 +101,10 @@ pub(crate) fn instant_candidate_span(
/// Selects source rows for an aligned PromQL instant-evaluation range.
///
/// Returns original logical row indices and evaluation timestamps from a complete source axis of
/// strictly increasing milliseconds. Any provided field's stale marker suppresses selection;
/// Returns original logical row indices and evaluation timestamps from a complete source axis
/// of strictly increasing milliseconds. Any provided field's stale marker suppresses selection;
/// missing samples are not stale and remain selectable.
///
/// The ordering of the axis is [`RawTimestamps`]' own guarantee, checked when it was built, so this
/// walks the axis once -- one cursor advance per row -- and validates nothing about it again.
///
/// The two vectors it returns hold one `u64` source row index and one `i64` evaluation timestamp
/// per selected point, so what it allocates is 16 bytes per point of the clipped grid rather than
/// anything proportional to the samples of the axis. It reserves none of that: every caller grows
@@ -197,7 +178,6 @@ pub fn select_instants(
};
current_stale |= stale;
}
row_read();
cursor += 1;
}
let Some(candidate) = cursor.checked_sub(1) else {
+2 -39
View File
@@ -15,7 +15,6 @@
use std::collections::BTreeMap;
use std::ops::RangeInclusive;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use datatypes::arrow::array::{Float64Array, UInt32Array};
use datatypes::arrow::buffer::ScalarBuffer;
@@ -29,8 +28,8 @@ use datatypes::series::samples::{
use datatypes::series::time::{EvalGrid, GridSpec};
use super::{
AXIS_ROWS_READ, InstantCandidateSpan, info_sample_times, instant_candidate_span,
select_instants, timestamp_of_evaluated, timestamp_of_selection,
InstantCandidateSpan, info_sample_times, instant_candidate_span, select_instants,
timestamp_of_evaluated, timestamp_of_selection,
};
use crate::grid_series::GridSeriesBuilder;
use crate::grid_series::reference::{Labels, labels_of, metric_of};
@@ -234,42 +233,6 @@ fn offset_extreme_timestamps_are_checked_without_large_fixture() {
assert_eq!(times, [1_000_000]);
}
/// A selection walks its axis once, whatever the grid it selects onto.
///
/// The strictly increasing order of the timestamps is `RawTimestamps`' guarantee, checked when the
/// axis is built, so the selector has nothing left to validate: the rows it reads are the rows of
/// the axis, once each, and a second pass over them -- such as a re-check of their order -- would
/// show up here as twice as many. The selection itself is the same either way, which is why the
/// count has to be what the test reads.
#[test]
fn a_selection_reads_its_axis_once() {
let rows = 5_000;
let times: Vec<i64> = (0..rows).map(|row| row as i64).collect();
let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0; rows])));
let read = |range: RangeInclusive<i64>, step: i64| -> u64 {
AXIS_ROWS_READ.store(0, Ordering::Relaxed);
let (selected, _) = select(&times, &fields, range, step, 10, 0).unwrap();
assert!(!selected.is_empty(), "the selection is what reads the axis");
AXIS_ROWS_READ.load(Ordering::Relaxed)
};
// Every row is selected: one pass, no more.
assert_eq!(
read(0..=rows as i64 - 1, 1),
rows as u64,
"the axis was walked more than once"
);
// Fewer points over the whole axis read it once as well: the pass follows the samples, not the
// points, so it never exceeds the rows of the axis however many points the grid holds.
for step in [10, 499] {
let visited = read(0..=rows as i64 - 1, step);
assert!(
visited <= rows as u64,
"step {step} read {visited} rows of a {} row axis",
rows
);
}
}
#[test]
fn instant_candidate_span_preserves_unaligned_and_wide_ranges() {
// The query end is not on the start's lattice. The scan bound is 125, but stepping from 100
+30 -166
View File
@@ -23,10 +23,10 @@
//!
//! # What decides it
//!
//! Only a rewrite can alias, and only one that writes a label its input already carries or drops one
//! its output would still be told apart by: [`duplicates_canonical_labels`] is that fact, derived
//! from the tree and the tags each source reports. A tree with no such rewrite gains no merge at
//! all, so a per-series query keeps every scan partition and pays nothing for this pass.
//! Only a rewrite can alias, and only one that writes a label its input already carries:
//! [`duplicates_canonical_labels`] is that fact, derived from the tree and the tags each source
//! reports. A tree with no such rewrite gains no merge at all, so a per-series query keeps every
//! scan partition and pays nothing for this pass.
//!
//! Where a merge is added is a property of the consumer, and of what the operations between the
//! consumer and the rewrite do to one sample:
@@ -45,39 +45,6 @@
//! A merge is never added below a raw scan, a scalar or aggregate state: it binds samples evaluated
//! on a grid, and no consumer asks for one over those.
//!
//! A node can be a source of duplicates without rewriting a label its input carries: a range
//! function that computes its result and `timestamp` in both of its forms drop the metadata labels -
//! the metric name, `__type__` and `__unit__` - and hand out a set those no longer tell apart. What
//! they produce is collected by their consumer - the root included - exactly as a rewrite of an
//! existing label is, unless
//! [`duplicates_canonical_labels`](crate::native::plan::duplicates_canonical_labels) proves that
//! nothing they drop was telling two of their series apart: a source bound to one metric names
//! every series it hands over, a source whose tags carry neither `__type__` nor `__unit__` holds no
//! series those two tell apart either, and the scan's key columns always decode a value.
//! `rate(some_metric[5m])` over such a scan is then a window that streams its scan, and the same
//! query over a selector that discovered several metrics keeps the collection it needs. The same
//! rule - the labels and the scan's keys - is what a math or histogram map, a parameterized map and
//! an arithmetic or `bool` scalar broadcast are judged by.
//!
//! The key half is not the tree's to answer, so the caller states it as `keys_survive`: two stored
//! series of one metric whose key column holds `NULL` in one and `''` in the other decode to an
//! absent and to an empty label, and the drop removes the empty one with the name, leaving both
//! under one label set. The rule is
//! [`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop),
//! which the physical per-series scan reads to decide whether a node may stream and the planner
//! reads from the resolved table, so the two cannot disagree about a drop. A caller that cannot
//! state it passes `false`; the statements collected for that reason alone are counted
//! ([`leaves_a_drop_to_the_storage_keys`](crate::native::plan::leaves_a_drop_to_the_storage_keys)).
//!
//! A drop above a merge is not asked about the empty value it removes, which is what keeps a chain of
//! drops to one collection: [`NativeExpr::Merge`] hands over one series per canonical label set - the
//! comparison that treats an empty label value as no label - so no drop above it can fold two of its
//! series by dropping an empty one. The metadata labels a drop removes are another matter: a merge
//! keeps two series that differ in `__name__`, `__type__` or `__unit__`, so the labels half of the
//! proof is still asked above it. With that, `abs(abs(rate(x[5m])))` over a table whose key columns
//! can spell the `NULL`/`''` pair pays for the one merge at the bottom of the chain rather than one
//! per drop.
//!
//! Canonicalizing a map's input is what makes the histogram case work: in
//! `histogram_count(label_replace(x, "host", "same", "host", ".*"))` two scan partitions each
//! rewrite one series to `{host="same"}` and `histogram_count` drops both - they are float samples -
@@ -94,8 +61,6 @@
//! execution is the lowering's question. A merge that spans partitions uses a producer and the
//! existing canonical router; downstream consumers read ordinary evaluated frames.
use std::sync::Arc;
use datafusion::error::Result as DfResult;
use crate::native::plan::{
@@ -112,20 +77,6 @@ mod tests;
/// source the caller says nothing about proves nothing, and a rewrite over it is collected. The
/// index is what a leaf reads, so a tree with several sources is judged source by source.
///
/// `source_metrics` is the other half of that knowledge, indexed the same way: the one metric each
/// source is bound to, or `None` when it may hand over several. It is what lets a kernel that drops
/// the metadata labels keep every partition - see
/// [`duplicates_canonical_labels`](crate::native::plan::duplicates_canonical_labels) - and a caller
/// that cannot say collects instead.
///
/// `keys_survive` is the last fact and the one the tree cannot carry: whether every scan below
/// `tree` still tells its series apart once a drop has normalized their labels
/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop),
/// read from the resolved table). A key column that holds `NULL` in one series and `''` in another
/// decodes to an absent and to an empty label, and the empty one leaves with the metadata labels, so
/// `false` collects a drop the labels alone would have proved. A caller that cannot state it passes
/// `false`.
///
/// Every operation binding and every `source_index` stays where it was, so the rewrite changes which
/// series a consumer reads and not what the query computes. The result is checked by
/// [`NativeTree::try_new`], because a merge has to sit over an evaluated vector and the pass may
@@ -133,12 +84,10 @@ mod tests;
pub fn merge_label_rewrites(
tree: &NativeTree,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> DfResult<NativeTree> {
let root = rewrite(tree.root(), source_tags, source_metrics, keys_survive)?;
let root = rewrite(tree.root(), source_tags)?;
// The root is a consumer too: whatever reads this tree reads one series per label set.
NativeTree::try_new(canonical(root, source_tags, source_metrics, keys_survive))
NativeTree::try_new(canonical(root, source_tags))
}
/// Wraps `node` in a [`NativeExpr::Merge`] when its pieces can share a label set.
@@ -151,19 +100,14 @@ pub fn merge_label_rewrites(
/// ([`duplicates_canonical_labels`]), because collecting pieces that are already distinct is a
/// barrier that buys nothing;
/// - it is not already a merge, which would collect what is already collected.
fn canonical(
node: NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> NativeNode {
fn canonical(node: NativeNode, source_tags: &[Vec<String>]) -> NativeNode {
if !matches!(&node.output, NativeType::Samples(schema) if schema.eval_grid.is_some()) {
return node;
}
if matches!(node.expr, NativeExpr::Merge { .. }) {
return node;
}
if !duplicates_canonical_labels(&node, source_tags, source_metrics, keys_survive) {
if !duplicates_canonical_labels(&node, source_tags) {
return node;
}
// A merge reports its input's layout: the same fields on the same grid.
@@ -178,12 +122,7 @@ fn canonical(
/// Rewrites `node` bottom-up: its children first, then the merge its own consumer contract needs of
/// them.
fn rewrite(
node: &NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> DfResult<NativeNode> {
fn rewrite(node: &NativeNode, source_tags: &[Vec<String>]) -> DfResult<NativeNode> {
let expr = match &node.expr {
// A source is the boundary it was bound to and there is nothing below it to rewrite. A
// constant reads nothing.
@@ -194,27 +133,22 @@ fn rewrite(
// grid; a timestamp replaces each value with its sample's time. Both read each piece as it
// is, so neither collects what it reads.
NativeExpr::Instant { input, op } => NativeExpr::Instant {
input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?),
input: Box::new(rewrite(input, source_tags)?),
op: op.clone(),
},
NativeExpr::Timestamp { input } => NativeExpr::Timestamp {
input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?),
input: Box::new(rewrite(input, source_tags)?),
},
// A merge is the canonicalizer itself, so it asks nothing of its input: fragments are what it
// was written to receive.
NativeExpr::Merge { input } => NativeExpr::Merge {
input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?),
input: Box::new(rewrite(input, source_tags)?),
},
// A window folds a range of one series' samples and hands the answer over as that series, so
// it reads the series whole. Over a raw scan the pieces are whole storage series already and
// the merge above is a no-op; over an evaluated vector a rewrite can have aliased them.
NativeExpr::Window { input, op } => NativeExpr::Window {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: op.clone(),
},
// A map is point-wise, but a point can drop out of its output: the histogram functions drop
@@ -223,12 +157,7 @@ fn rewrite(
// merge above this map would be handed an occupancy the map may already have erased - the
// pieces have to be collected where they still hold their samples, below it.
NativeExpr::Map { input, op } => NativeExpr::Map {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: op.clone(),
},
// A parameterized map's parameters are scalars, rewritten but never collected - a scalar
@@ -239,38 +168,23 @@ fn rewrite(
op,
parameters,
} => NativeExpr::ParameterizedMap {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: *op,
parameters: parameters
.iter()
.map(|parameter| rewrite(parameter, source_tags, source_metrics, keys_survive))
.map(|parameter| rewrite(parameter, source_tags))
.collect::<DfResult<Vec<_>>>()?,
},
// A fold answers its groups, so two pieces of one series in a group are summed as if the
// query had written them twice: the merge is what makes one series arrive as one.
NativeExpr::Aggregate { input, op } => NativeExpr::Aggregate {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: op.clone(),
},
// Vector operations consume canonical series: ranking counts them, and presence checks and
// folds must not silently consume overlapping fragments that a merge would reject.
NativeExpr::Vector { input, op } => NativeExpr::Vector {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: op.clone(),
},
// Each side of a binary or set operation is matched by label on its own, so each is
@@ -281,33 +195,13 @@ fn rewrite(
right,
matching,
} => NativeExpr::Binary {
left: Box::new(canonical(
rewrite(left, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
right: Box::new(canonical(
rewrite(right, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
left: Box::new(canonical(rewrite(left, source_tags)?, source_tags)),
right: Box::new(canonical(rewrite(right, source_tags)?, source_tags)),
matching: matching.clone(),
},
NativeExpr::Set { left, right, op } => NativeExpr::Set {
left: Box::new(canonical(
rewrite(left, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
right: Box::new(canonical(
rewrite(right, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
left: Box::new(canonical(rewrite(left, source_tags)?, source_tags)),
right: Box::new(canonical(rewrite(right, source_tags)?, source_tags)),
op: op.clone(),
},
// `info` pairs each base series with the info series that share its identifying labels, so it
@@ -318,33 +212,18 @@ fn rewrite(
info,
selector,
} => NativeExpr::Info {
base: Box::new(canonical(
rewrite(base, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
info: Box::new(canonical(
rewrite(info, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
base: Box::new(canonical(rewrite(base, source_tags)?, source_tags)),
info: Box::new(canonical(rewrite(info, source_tags)?, source_tags)),
selector: selector.clone(),
},
// `vector(s)` builds its series from a scalar, which carries no label set to collect.
NativeExpr::VectorOf { input } => NativeExpr::VectorOf {
input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?),
input: Box::new(rewrite(input, source_tags)?),
},
// `scalar()` answers from the whole vector: a series that arrives in pieces would be seen
// as several samples, so the pieces are collected where they still hold their samples.
NativeExpr::ScalarOf { input } => NativeExpr::ScalarOf {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
},
// `topk` ranks the whole vector, so it counts canonical series; its parameter is a scalar,
// rewritten but never collected - a scalar carries no label set.
@@ -353,19 +232,9 @@ fn rewrite(
op,
parameter,
} => NativeExpr::ParameterizedTopK {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: op.clone(),
parameter: Box::new(rewrite(
parameter,
source_tags,
source_metrics,
keys_survive,
)?),
parameter: Box::new(rewrite(parameter, source_tags)?),
},
// A histogram function folds the bucket series of one histogram, so overlapping fragments
// must not reach it; its parameters are scalars and only need rewriting.
@@ -374,16 +243,11 @@ fn rewrite(
op,
parameters,
} => NativeExpr::ParameterizedHistogram {
input: Box::new(canonical(
rewrite(input, source_tags, source_metrics, keys_survive)?,
source_tags,
source_metrics,
keys_survive,
)),
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
op: op.clone(),
parameters: parameters
.iter()
.map(|parameter| rewrite(parameter, source_tags, source_metrics, keys_survive))
.map(|parameter| rewrite(parameter, source_tags))
.collect::<DfResult<Vec<_>>>()?,
},
};
+41 -772
View File
@@ -252,31 +252,6 @@ fn set_over(left: NativeNode, right: NativeNode, out: Arc<EvalGrid>) -> NativeNo
}
}
/// `last_over_time(inner[5m])`, a window that folds `inner` onto `out` and keeps the metric name.
fn keeping_window_over(
inner: NativeNode,
input_grid: Option<Arc<EvalGrid>>,
out: Arc<EvalGrid>,
) -> NativeNode {
NativeNode {
expr: NativeExpr::Window {
input: Box::new(inner),
op: BoundWindow {
function: RangeFunction::LastOverTime,
window: RangeWindow {
range_ms: 300_000,
offset_ms: 0,
anchor_ms: None,
modifier: None,
},
input_grid,
output_grid: Arc::clone(&out),
},
},
output: evaluated(fields(1), out),
}
}
/// `rate(inner[5m])`, a window that folds `inner` onto `out` and drops the metric name.
fn window_over(
inner: NativeNode,
@@ -353,54 +328,9 @@ fn merges(tree: &NativeTree) -> usize {
}
/// Runs the pass over `root`, whose sources report `source_tags`.
///
/// Nothing is stated about which metric each source reads, so a kernel that drops the metadata
/// labels over one of them proves nothing and is collected.
fn passed(root: NativeNode, source_tags: &[Vec<String>]) -> NativeTree {
passed_over(root, source_tags, &[])
}
/// Runs the pass over `root`, whose sources report `source_tags` and read the metric each of
/// `source_metrics` names, indexed the way `merge_label_rewrites` takes it.
///
/// The scans below these fixtures key their series by columns that always decode a value, which is
/// the case a metadata drop has to be proved against the labels alone: `NULL` and `''` in one key
/// column are what [`passed_over_nullable_keys`] states instead.
fn passed_over(
root: NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
) -> NativeTree {
passed_over_with_keys(root, source_tags, source_metrics, true)
}
/// The same pass over a tree whose scans key their series by a column that can hold `NULL` in one
/// series and `''` in another, so no metadata drop below it can be proved by the labels alone.
fn passed_over_nullable_keys(
root: NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
) -> NativeTree {
passed_over_with_keys(root, source_tags, source_metrics, false)
}
/// Runs the pass over `root` with the key fact the caller states, which is what
/// `merge_label_rewrites` takes as `keys_survive`.
fn passed_over_with_keys(
root: NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> NativeTree {
let tree = NativeTree::try_new(root).expect("the fixture is a legal tree");
merge_label_rewrites(&tree, source_tags, source_metrics, keys_survive)
.expect("the pass produces a legal tree")
}
/// The labels a scan bound to one metric reports: its tag, and the name the binding gives every
/// series it hands over, which is not a tag column.
fn named_scan_tags() -> Vec<Vec<String>> {
vec![tags(&["host", "__name__"])]
merge_label_rewrites(&tree, source_tags).expect("the pass produces a legal tree")
}
fn tags(names: &[&str]) -> Vec<String> {
@@ -508,7 +438,7 @@ fn presence_and_folding_vector_operations_collect_aliasing_inputs() {
};
assert!(matches!(input.expr, NativeExpr::Map { .. }), "{op:?}");
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(1, merges(&twice), "{op:?}");
assert_eq!(tree.to_string(), twice.to_string(), "{op:?}");
}
@@ -563,7 +493,7 @@ fn rewrites_after_ranking_or_grouping_are_collected() {
),
"{name}"
);
let twice = merge_label_rewrites(&tree, &source_tags, &[], true).unwrap();
let twice = merge_label_rewrites(&tree, &source_tags).unwrap();
assert_eq!(tree.to_string(), twice.to_string(), "{name}");
assert_eq!(1, merges(&twice), "{name}");
}
@@ -574,8 +504,7 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() {
let out = grid(0, 1000, 2);
// A match answers one series per label set, but those labels are its operands': renaming `host`
// to a constant above it puts both sides' results under one set, so the consumer above the
// rewrite is the one that has to collect them. The match itself reads its operands as they are;
// the one on the right drops the metadata labels, so its own answer is collected.
// rewrite is the one that has to collect them. The match itself reads its operands as they are.
let matched = binary_over(selected(&out), abs_map(selected(&out)), Arc::clone(&out));
let renamed = map_ops(
matched,
@@ -584,11 +513,7 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() {
))],
);
let tree = passed(renamed, &scan_tags());
assert_eq!(
2,
merges(&tree),
"the rewrite's own collection, and the one the name-dropping operand hands out: {tree}"
);
assert_eq!(1, merges(&tree));
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the statement must collect the rewrite above the match");
};
@@ -599,9 +524,9 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() {
matches!(input.expr, NativeExpr::Binary { .. }),
"the match is not handed the rewritten operand"
);
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(tree.to_string(), twice.to_string());
assert_eq!(2, merges(&twice));
assert_eq!(1, merges(&twice));
}
#[test]
@@ -661,21 +586,18 @@ fn a_pointwise_map_reads_each_piece_as_it_is() {
abs_map(rewrite(selected(&out), "host", "host")),
&scan_tags(),
);
// Two collections: one below the map, because a map reads pieces point-wise and a point it
// reads can be one it drops, and one above it, for the metadata labels the map itself drops.
assert_eq!(2, merges(&tree), "{tree}");
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the map hands out: {tree}");
};
let NativeExpr::Map { input, .. } = &input.expr else {
assert_eq!(1, merges(&tree));
// The map stays the root and its input is the merge: a map reads pieces point-wise, but a point
// it reads can be one it drops, so the collection has to happen below it.
let NativeExpr::Map { input, .. } = &tree.root().expr else {
panic!("the root stays the point-wise map");
};
let NativeExpr::Merge { input } = &input.expr else {
panic!("the map is handed one series per label set");
};
assert!(matches!(input.expr, NativeExpr::Map { .. }));
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
assert_eq!(2, merges(&twice));
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(1, merges(&twice));
assert_eq!(tree.to_string(), twice.to_string());
}
@@ -691,15 +613,8 @@ fn a_sample_dropping_map_collects_below_itself() {
))],
);
let tree = passed(histogram_count(aliasing), &scan_tags());
assert_eq!(
2,
merges(&tree),
"one merge below the sample-dropping map, one for the labels it drops"
);
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what `histogram_count` hands out: {tree}");
};
let NativeExpr::Map { input, .. } = &input.expr else {
assert_eq!(1, merges(&tree), "one merge, below the sample-dropping map");
let NativeExpr::Map { input, .. } = &tree.root().expr else {
panic!("the root stays `histogram_count`");
};
let NativeExpr::Merge { input } = &input.expr else {
@@ -718,51 +633,36 @@ fn a_sample_dropping_map_collects_below_itself() {
);
// A second pass over the collected tree changes nothing: the merge answers one series per label
// set, so nothing below the map aliases any more.
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
assert_eq!(2, merges(&twice));
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(1, merges(&twice));
assert_eq!(tree.to_string(), twice.to_string());
}
#[test]
fn a_pointwise_map_collects_its_own_name_drop() {
fn a_pointwise_map_over_a_clean_input_gains_no_merge() {
let out = grid(0, 1000, 2);
// No rewrite anywhere in the tree, so no two pieces were told apart by a label one of them
// wrote - but `abs` drops the metadata labels, and the drop is a source of duplicates of its
// own: a key column that holds `NULL` in one stored series and `''` in another decodes to an
// absent and to an empty label, and both are gone once the empty one goes. Each map's answer is
// therefore collected, and both sit above the map that hands them out.
// No rewrite anywhere in the tree, so no two pieces can share a label set.
let tree = passed(abs_map(abs_map(selected(&out))), &scan_tags());
assert_eq!(2, merges(&tree), "{tree}");
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the outer map hands out: {tree}");
assert_eq!(0, merges(&tree));
let NativeExpr::Map { input, .. } = &tree.root().expr else {
panic!("the root stays the point-wise map");
};
let NativeExpr::Map { input, .. } = &input.expr else {
panic!("the merge sits above the map, not below it: {tree}");
};
let NativeExpr::Merge { input } = &input.expr else {
panic!("the outer map reads one series per label set: {tree}");
};
assert!(matches!(input.expr, NativeExpr::Map { .. }), "{tree}");
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
assert_eq!(2, merges(&twice));
assert!(matches!(input.expr, NativeExpr::Map { .. }));
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(tree.to_string(), twice.to_string());
}
#[test]
fn a_sample_dropping_map_over_a_fresh_destination_collects_its_own_drop() {
fn a_sample_dropping_map_over_a_fresh_destination_gains_no_merge() {
let out = grid(0, 1000, 2);
// `dst` names no label the scan reports, so the rewrite cannot alias and the map's input needs
// no collection even though the map drops samples - but the metadata labels `histogram_count`
// drops are a source of duplicates of their own, so its answer is collected.
// no collection even though the map drops samples.
let tree = passed(
histogram_count(rewrite(selected(&out), "dst", "host")),
&scan_tags(),
);
assert_eq!(1, merges(&tree), "a fresh destination cannot alias: {tree}");
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what `histogram_count` hands out: {tree}");
};
let NativeExpr::Map { input, .. } = &input.expr else {
assert_eq!(0, merges(&tree), "a fresh destination cannot alias");
let NativeExpr::Map { input, .. } = &tree.root().expr else {
panic!("the root stays `histogram_count`");
};
assert!(matches!(input.expr, NativeExpr::Map { .. }));
@@ -780,15 +680,10 @@ fn a_parameterized_map_collects_below_itself() {
),
&scan_tags(),
);
// One below the map, for the point a clamp can empty, and one above it, for the metadata labels
// the kernel drops.
assert_eq!(2, merges(&tree), "{tree}");
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the clamp hands out: {tree}");
};
assert_eq!(1, merges(&tree));
let NativeExpr::ParameterizedMap {
input, parameters, ..
} = &input.expr
} = &tree.root().expr
else {
panic!("the root stays the parameterized map");
};
@@ -801,8 +696,8 @@ fn a_parameterized_map_collects_below_itself() {
matches!(parameters[0].expr, NativeExpr::Scalar(_)),
"a scalar argument is not collected"
);
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
assert_eq!(2, merges(&twice));
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(1, merges(&twice));
assert_eq!(tree.to_string(), twice.to_string());
}
@@ -814,16 +709,12 @@ fn each_side_of_a_match_is_collected_on_its_own() {
binary_over(aliasing(), abs_map(selected(&out)), Arc::clone(&out)),
&scan_tags(),
);
assert_eq!(
2,
merges(&tree),
"the aliasing side and the name-dropping side are each collected: {tree}"
);
assert_eq!(1, merges(&tree), "only the aliasing side needs collecting");
let NativeExpr::Binary { left, right, .. } = &tree.root().expr else {
panic!("the root stays the match");
};
assert!(matches!(left.expr, NativeExpr::Merge { .. }));
assert!(matches!(right.expr, NativeExpr::Merge { .. }));
assert!(!matches!(right.expr, NativeExpr::Merge { .. }));
}
#[test]
@@ -845,10 +736,9 @@ fn a_set_operation_collects_each_side() {
#[test]
fn a_window_collects_an_aliasing_child() {
let out = grid(0, 1000, 2);
// A window over an evaluated vector folds one series' range, so it reads the series whole. The
// window keeps the metric name here, so the rewrite below it is the only source of duplicates.
// A window over an evaluated vector folds one series' range, so it reads the series whole.
let tree = passed(
keeping_window_over(
window_over(
rewrite(selected(&out), "host", "host"),
Some(Arc::clone(&out)),
Arc::clone(&out),
@@ -866,614 +756,10 @@ fn a_window_collects_an_aliasing_child() {
fn a_window_over_a_raw_scan_collects_nothing() {
let out = grid(0, 1000, 2);
// The scan hands over whole storage series, and a merge cannot take raw samples anyway.
let tree = passed(keeping_window_over(raw_input(0), None, out), &scan_tags());
let tree = passed(window_over(raw_input(0), None, out), &scan_tags());
assert_eq!(0, merges(&tree));
}
#[test]
fn a_name_dropping_window_is_collected_by_its_consumer() {
let out = grid(0, 1000, 2);
// `rate` drops the metric name, which is part of the identity a merge groups by: a caller that
// says nothing about which metric the scan reads collects what the window hands out.
let tree = passed(
window_over(raw_input(0), None, Arc::clone(&out)),
&named_scan_tags(),
);
assert_eq!(1, merges(&tree));
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the window hands out: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Window { .. }),
"the merge sits above the window, not below it"
);
}
#[test]
fn a_name_dropping_window_over_one_metric_streams() {
let out = grid(0, 1000, 2);
// The scan is bound to one metric, so every series it hands over is named the same thing:
// dropping that name tells none of them apart, and the window keeps every partition.
//
// The key columns are the other half of that proof, and this fixture states the half that holds:
// every one of them always decodes a value, so the empty label value the drop removes with the
// name never stood for a second stored series.
let tree = passed_over(
window_over(raw_input(0), None, Arc::clone(&out)),
&named_scan_tags(),
&[Some(Arc::from("some_metric"))],
);
assert_eq!(0, merges(&tree), "{tree}");
assert!(
matches!(tree.root().expr, NativeExpr::Window { .. }),
"the root stays the window: {tree}"
);
}
/// Two stored series of one metric that differ only in a tag column holding `NULL` in one and `''`
/// in the other: `{host="web-1", zone=""}` and `{host="web-1"}` are two series to the scan, and
/// `rate` drops the metric name and the empty `zone` with it, so both leave the window as
/// `{host="web-1"}`.
///
/// The labels alone cannot tell them apart - they share host, zone is about to go, and the name is
/// the one every series of the scan shares - so the scan's key columns are what decides the answer.
/// A key column that can hold that pair is the fact that says so, and the drop is collected: the
/// merge is where the second value of one label set at one position is refused, which is
/// Prometheus's answer to two series of one label set rather than two pieces of one series.
///
/// The execution half of this - that the merge refuses the pair instead of handing both over - is
/// `two_series_onto_one_label_set_after_a_window_are_rejected` in `native::execute::tests`.
#[test]
fn a_nullable_key_tag_collects_the_window_that_drops_an_empty_value() {
let out = grid(0, 1000, 2);
// A scan of one metric whose tags are `host` and `zone`, the second of which is a nullable key
// column: the pair above is one storage key away from being two.
let scan = [tags(&["host", "zone", "__name__"])];
let single = [Some(Arc::<str>::from("some_metric"))];
let tree = passed_over_nullable_keys(
window_over(raw_input(0), None, Arc::clone(&out)),
&scan,
&single,
);
assert!(
merges(&tree) >= 1,
"a nullable key column keeps the collection `rate` needs: {tree}"
);
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the window hands out: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Window { .. }),
"the merge sits above the window: {tree}"
);
// The same tree over a scan whose keys always decode a value: the labels prove the drop, and
// the window keeps every partition.
let tree = passed_over(
window_over(raw_input(0), None, Arc::clone(&out)),
&scan,
&single,
);
assert_eq!(
0,
merges(&tree),
"non-nullable keys keep the proof, so nothing is collected: {tree}"
);
}
#[test]
fn a_name_dropping_window_over_several_metrics_is_collected() {
let out = grid(0, 1000, 2);
// A selector that discovered several metrics binds no single name, so two of its series can be
// told apart by the name alone - and dropping it folds them into one label set.
let tree = passed_over(
window_over(raw_input(0), None, Arc::clone(&out)),
&named_scan_tags(),
&[None],
);
assert_eq!(1, merges(&tree), "{tree}");
assert!(
matches!(tree.root().expr, NativeExpr::Merge { .. }),
"the root collects what the window hands out: {tree}"
);
}
#[test]
fn a_metadata_tag_in_the_binding_fails_closed() {
let out = grid(0, 1000, 2);
// `__type__` and `__unit__` go with the name, and a scan that carries one of them as a tag can
// hold two series that differ only there. A fixed metric name proves nothing about them, so the
// drop stays collected.
for metadata in ["__type__", "__unit__"] {
let scan = vec![tags(&["host", "__name__", metadata])];
let tree = passed_over(
window_over(raw_input(0), None, Arc::clone(&out)),
&scan,
&[Some(Arc::from("some_metric"))],
);
assert_eq!(1, merges(&tree), "{metadata}: {tree}");
}
}
/// The name a name-dropping kernel removes is the one every series of a single-metric scan shares,
/// and nothing else it drops tells two of them apart, so a math map, a histogram map, a
/// parameterized math kernel and an arithmetic scalar broadcast handed such a source keep every
/// partition.
///
/// The scan's key columns are the other half of that proof, and this is the case they hold for:
/// every one of them always decodes a value, so the empty label value the drop removes with the name
/// never stood for a second stored series. [`a_name_dropping_kernel_over_a_nullable_key_is_collected`]
/// is the same fixtures over a scan whose keys can spell that pair.
#[test]
fn a_name_dropping_kernel_over_one_metric_keeps_its_partitions() {
let out = grid(0, 1000, 2);
let single = [Some(Arc::<str>::from("some_metric"))];
let kept = |name: &str, root: NativeNode| {
let tree = passed_over(root, &named_scan_tags(), &single);
assert_eq!(0, merges(&tree), "`{name}` cannot alias: {tree}");
assert!(
!matches!(tree.root().expr, NativeExpr::Merge { .. }),
"`{name}` hands over one series per label set already: {tree}"
);
};
// A math map.
kept("abs", abs_map(selected(&out)));
// A histogram map.
kept("histogram_count", histogram_count(selected(&out)));
// A parameterized math kernel.
kept(
"clamp_min",
clamp_min(selected(&out), scalar(0.0, Arc::clone(&out))),
);
// An arithmetic scalar broadcast, `x * 2`.
kept(
"x * 2",
binary_over(
selected(&out),
scalar(2.0, Arc::clone(&out)),
Arc::clone(&out),
),
);
}
/// The same kernels over a binding that carries `__type__` or `__unit__` as a tag are collected at
/// the root.
///
/// Those two go with the name when a kernel drops it, so two series that differ only there meet once
/// the drop removes them, whatever the scan's key columns read otherwise: this is the labels half
/// failing, not the key half. It is asked before the key columns are.
#[test]
fn a_name_dropping_kernel_over_a_metadata_tag_is_collected() {
let out = grid(0, 1000, 2);
let single = [Some(Arc::<str>::from("some_metric"))];
let collected = |name: &str, tags: &[Vec<String>], root: NativeNode| {
let tree = passed_over(root, tags, &single);
assert_eq!(1, merges(&tree), "`{name}` drops the name: {tree}");
assert!(
matches!(tree.root().expr, NativeExpr::Merge { .. }),
"`{name}` hands out one series per label set: {tree}"
);
};
for metadata in ["__type__", "__unit__"] {
let scan = vec![tags(&["host", "__name__", metadata])];
collected("abs", &scan, abs_map(selected(&out)));
collected("histogram_count", &scan, histogram_count(selected(&out)));
collected(
"clamp_min",
&scan,
clamp_min(selected(&out), scalar(0.0, Arc::clone(&out))),
);
collected(
"x * 2",
&scan,
binary_over(
selected(&out),
scalar(2.0, Arc::clone(&out)),
Arc::clone(&out),
),
);
}
}
/// A name-dropping kernel above another one asks for no collection of its own, and neither does a
/// broadcast above a window that drops the name: `rate(x[5m]) * 2` plans one window and one
/// broadcast, and `abs(abs(rate(x[5m])))` one window and two maps, with no merge in either.
///
/// That is the answer over a scan whose key columns always decode a value. Over one that can hold
/// `NULL` in one series and `''` in the other, the drop the labels cannot prove is collected - but
/// only once, at the bottom of the chain: the merge answers the empty-value half of every drop above
/// it, because it hands over one series per label set under the comparison that treats an empty
/// value as no label. Each extra merge would buffer its whole input to collect a pair the drop above
/// it cannot fold. The labels half is a different matter and is still asked - see
/// [`a_merge_below_a_drop_answers_only_the_empty_values`].
#[test]
fn a_drop_above_a_name_dropping_kernel_needs_no_collection() {
let out = grid(0, 1000, 2);
// What the planner states for a scan of one metric: its tag columns and the name the binding
// gives every series it hands over.
let scan = [tags(&["host", "zone", "__name__"])];
let single = [Some(Arc::<str>::from("some_metric"))];
let rate = || window_over(raw_input(0), None, Arc::clone(&out));
// `rate(x[5m]) * 2`: the broadcast drops the metadata labels the window dropped already.
let broadcast = binary_over(rate(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out));
// `abs(abs(rate(x[5m])))`: two math kernels above the window, neither of which can fold a pair
// the window left distinct.
let nested = abs_map(abs_map(rate()));
for (query, root) in [
("rate(x[5m]) * 2", broadcast),
("abs(abs(rate(x[5m])))", nested),
] {
let tree = passed_over(root, &scan, &single);
assert_eq!(0, merges(&tree), "`{query}` plans a merge: {tree}");
}
}
/// The same two queries over a scan whose keys can spell one series as `NULL` and another as `''`:
/// the drop is collected once, at the bottom of the chain, and the drops above it cost nothing.
///
/// One merge is what a nullable key column costs a statement, whatever the length of the chain above
/// it. The merge sits above the window - the bottom-most drop - so every map and broadcast above it
/// is asked nothing: what they read is already one series per canonical label set.
#[test]
fn a_nullable_key_column_collects_a_chain_of_drops_once() {
let out = grid(0, 1000, 2);
// The same facts as the test above, with a key column that can hold `NULL` in one series and
// `''` in the other, which no labels can prove apart.
let scan = [tags(&["host", "zone", "__name__"])];
let single = [Some(Arc::<str>::from("some_metric"))];
let rate = || window_over(raw_input(0), None, Arc::clone(&out));
let broadcast = binary_over(rate(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out));
let nested = abs_map(abs_map(rate()));
for (query, root) in [
("rate(x[5m]) * 2", broadcast),
("abs(abs(rate(x[5m])))", nested),
] {
let tree = passed_over_nullable_keys(root, &scan, &single);
assert_eq!(
1,
merges(&tree),
"`{query}` pays for the one merge its key columns need, not one per drop: {tree}"
);
}
}
/// A nullable key column collects the drop it is about, and that merge sits above the drop rather
/// than anywhere else in the chain: the window's answer is what the two stored series arrive at.
#[test]
fn a_nullable_key_column_collects_the_bottom_most_drop() {
let out = grid(0, 1000, 2);
let scan = [tags(&["host", "zone", "__name__"])];
let single = [Some(Arc::<str>::from("some_metric"))];
// `abs(abs(rate(x[5m])))` over the nullable key: the merge is the window's consumer, and both
// maps above it read what it hands over.
let tree = passed_over_nullable_keys(
abs_map(abs_map(window_over(raw_input(0), None, Arc::clone(&out)))),
&scan,
&single,
);
assert_eq!(1, merges(&tree), "{tree}");
let NativeExpr::Map { input, .. } = &tree.root().expr else {
panic!("the root stays the outer map: {tree}");
};
let NativeExpr::Map { input, .. } = &input.expr else {
panic!("the two maps are not collected apart: {tree}");
};
let NativeExpr::Merge { input } = &input.expr else {
panic!("the merge sits above the window: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Window { .. }),
"the merge sits above the window, not below it: {tree}"
);
}
/// A merge below a drop answers the empty-value half of the proof only: a merge keeps two series
/// that differ in `__name__`, `__type__` or `__unit__`, and removing those is exactly what a
/// metadata drop does. Two metrics whose ordinary labels are equal are therefore still collected
/// above a merge, one merge per collision.
///
/// A selector that discovered several metrics is that shape: each metric's window is collected for
/// the name it drops, and the map above the merge is collected again for the name the merge kept.
#[test]
fn a_merge_below_a_drop_answers_only_the_empty_values() {
let out = grid(0, 1000, 2);
let scan = [tags(&["host", "__name__"])];
// The source may hand over several metrics, which is what leaves the labels half unproved above
// the window's merge.
let tree = passed_over(
abs_map(window_over(raw_input(0), None, Arc::clone(&out))),
&scan,
&[None],
);
assert_eq!(
2,
merges(&tree),
"the window's name drop and the map's are two collisions, and the merge between them \
cannot fold the names they remove: {tree}"
);
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the map hands out: {tree}");
};
let NativeExpr::Map { input, .. } = &input.expr else {
panic!("the root stays the map: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Merge { .. }),
"the window's answer is collected below the map: {tree}"
);
}
/// A `Merge` hands over one series per canonical label set - the one that treats an empty label
/// value as no label - so no drop above it can fold two of its series by removing an empty value.
/// The kernels above a merge therefore need no collection of their own for the empty-value half, one
/// merge per collision rather than one per kernel that repeats the collapse; the labels half is a
/// different matter (`a_merge_below_a_drop_answers_only_the_empty_values`).
#[test]
fn a_drop_above_a_merge_needs_no_collection_of_its_own() {
let out = grid(0, 1000, 2);
let single = [Some(Arc::<str>::from("some_metric"))];
// `label_replace(x, "host", "$1", "host", "(.*)")`: every host is renamed to one value, so the
// rewrite folds the scan's series into one label set and the pass collects it there.
let aliasing = || rewrite(selected(&out), "host", "host");
// `label_replace(x, "host", "$1", "host", "(.*)") * 2`: the broadcast drops the metadata labels
// above a merge, which cannot fold two series the merge already told apart.
let tree = passed_over(
binary_over(aliasing(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)),
&named_scan_tags(),
&single,
);
assert_eq!(1, merges(&tree), "`label_replace(...) * 2`: {tree}");
let NativeExpr::Binary { left, .. } = &tree.root().expr else {
panic!("the root stays the broadcast: {tree}");
};
let NativeExpr::Merge { input } = &left.expr else {
panic!("the vector side is handed one series per label set: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Map { .. }),
"the merge sits above the rewrite, and no second one above the match: {tree}"
);
// `abs(abs(rate(label_replace(...)[5m])))`: the rewrite's collision is the one collection, and
// neither map above it asks for another - their own name drop is over series a merge holds.
let tree = passed_over(
abs_map(abs_map(window_over(
aliasing(),
Some(Arc::clone(&out)),
Arc::clone(&out),
))),
&named_scan_tags(),
&single,
);
assert_eq!(
1,
merges(&tree),
"`abs(abs(rate(label_replace(...)[5m])))`: {tree}"
);
let NativeExpr::Map { input, .. } = &tree.root().expr else {
panic!("the root stays the outer map: {tree}");
};
let NativeExpr::Map { input, .. } = &input.expr else {
panic!("the two maps are not collected apart: {tree}");
};
let NativeExpr::Window { input, .. } = &input.expr else {
panic!("the inner map reads the window: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Merge { .. }),
"the one merge sits above the rewrite: {tree}"
);
// A second pass changes nothing: the merge answers one series per label set, and nothing above
// it collects again.
let twice = merge_label_rewrites(&tree, &named_scan_tags(), &single, true).unwrap();
assert_eq!(1, merges(&twice), "{twice}");
assert_eq!(tree.to_string(), twice.to_string());
}
/// A collection below a merge still happens: two series the tree folds into one label set there are
/// collected by the merge the pass puts above the rewrite, and the drop above them asks for nothing
/// more - the merge is what makes the two arrive as one.
#[test]
fn a_collision_below_a_merge_is_collected_there() {
let out = grid(0, 1000, 2);
let single = [Some(Arc::<str>::from("some_metric"))];
// `rate(label_replace(x, "host", "$1", "host", "(.*)")[5m])`: every host is renamed to one
// value, so the rewrite folds the scan's series into one label set - below the window, whose
// `rate` drops the metric name on top of that.
let tree = passed_over(
window_over(
rewrite(selected(&out), "host", "host"),
Some(Arc::clone(&out)),
Arc::clone(&out),
),
&named_scan_tags(),
&single,
);
assert_eq!(
1,
merges(&tree),
"the rewrite's own collection is what collects the collision: {tree}"
);
let NativeExpr::Window { input, .. } = &tree.root().expr else {
panic!("the root stays the window: {tree}");
};
let NativeExpr::Merge { input } = &input.expr else {
panic!("the window reads the collected rewrite: {tree}");
};
assert!(
matches!(input.expr, NativeExpr::Map { .. }),
"the merge sits above the rewrite, not above the window: {tree}"
);
let twice = merge_label_rewrites(&tree, &named_scan_tags(), &single, true).unwrap();
assert_eq!(1, merges(&twice), "{twice}");
assert_eq!(tree.to_string(), twice.to_string());
}
/// A name write is collected even when a merge already holds the series below it: rewriting a name
/// can still put two metrics whose ordinary labels are equal under one identity, which is the rule
/// the kernels' name-drop proof states.
///
/// `abs(label_replace(label_replace(x, "__name__", "$1", "host", "(.*)"), "host", "", "", ""))`
/// deletes the host the name was derived from, so the name is the only label still telling two
/// series apart. `single_metric_name` in `native::plan` is what refuses to carry a written name: it
/// would otherwise hand the scan's own name back, and a binding whose key columns always decode a
/// value would then make the drop look injective.
#[test]
fn a_written_name_is_not_the_name_the_scan_binds() {
let out = grid(0, 1000, 2);
// The scan reads one metric, so nothing but the written name is left to tell two of its series
// apart.
let single = [Some(Arc::<str>::from("some_metric"))];
let rewritten = map_ops(
map_ops(
selected(&out),
vec![MapOp::Label(Arc::new(
LabelRewrite::replace("__name__", "$1", "host", "(.*)").unwrap(),
))],
),
vec![MapOp::Label(Arc::new(
LabelRewrite::replace("host", "", "", "").unwrap(),
))],
);
let tree = passed_over(abs_map(rewritten), &named_scan_tags(), &single);
// The rewrite of the name and the rewrite of the `host` it was derived from are collected in
// their own right, and the root merge is the `abs` above them - the node whose answer the
// destination check decides.
assert_eq!(3, merges(&tree), "{tree}");
let NativeExpr::Merge { input: abs_node } = &tree.root().expr else {
panic!("the root collects what `abs` hands out: {tree}");
};
let NativeExpr::Map { input: below, op } = &abs_node.expr else {
panic!("the collected node is the map above the rewrites: {tree}");
};
assert!(
op.ops.iter().any(|op| matches!(op, MapOp::Math(_))),
"the root merge sits above `abs`, not above the rewrites: {tree}"
);
let NativeExpr::Merge { input: delete } = &below.expr else {
panic!("`abs` reads the collected rewrite: {tree}");
};
let NativeExpr::Map {
input: name_write, ..
} = &delete.expr
else {
panic!("the merge below `abs` sits above the delete: {tree}");
};
assert!(
matches!(name_write.expr, NativeExpr::Merge { .. }),
"the name write is collected below the delete: {tree}"
);
}
#[test]
fn a_timestamp_is_collected_by_its_consumer() {
let out = grid(0, 1000, 2);
// `timestamp` reports a sample's own time under the metric name dropped, so a caller that says
// nothing about the scan's metric collects its answer at the root as well.
let stamped = || NativeNode {
expr: NativeExpr::Timestamp {
input: Box::new(selected(&out)),
},
output: evaluated(fields(1), Arc::clone(&out)),
};
let tree = passed(stamped(), &named_scan_tags());
assert_eq!(1, merges(&tree));
assert!(
matches!(tree.root().expr, NativeExpr::Merge { .. }),
"the root collects what the timestamp hands out: {tree}"
);
// The scan reads one metric, so the name it drops told none of its series apart: the timestamp
// keeps every partition.
let tree = passed_over(
stamped(),
&named_scan_tags(),
&[Some(Arc::from("some_metric"))],
);
assert_eq!(0, merges(&tree), "{tree}");
assert!(
matches!(tree.root().expr, NativeExpr::Timestamp { .. }),
"the root stays the timestamp: {tree}"
);
}
/// The three nodes that drop the metadata labels on their own are judged by the labels: one metric
/// name over every series, and no metadata tag among them.
///
/// A name-dropping window, `timestamp(x)` and a `timestamp` over an evaluated input all drop the
/// metadata labels. A source bound to one metric names every series it hands over, so that name told
/// none of them apart and the drop is proved; a source that may hand over several is collected, and
/// so is one that carries `__type__` or `__unit__` among its tags. A window that keeps the name
/// drops nothing and hands its input's identity on.
#[test]
fn a_name_dropping_node_is_proved_by_the_labels_alone() {
let out = grid(0, 1000, 2);
let named = named_scan_tags();
let single = [Some(Arc::<str>::from("some_metric"))];
let collected =
|name: &str, tags: &[Vec<String>], metrics: &[Option<Arc<str>>], root: NativeNode| {
let tree = passed_over(root, tags, metrics);
assert_eq!(
1,
merges(&tree),
"`{name}` drops the metadata labels: {tree}"
);
assert!(
matches!(tree.root().expr, NativeExpr::Merge { .. }),
"the root collects `{name}`: {tree}"
);
};
let kept = |name: &str, root: NativeNode| {
let tree = passed_over(root, &named, &single);
assert_eq!(0, merges(&tree), "`{name}` cannot alias: {tree}");
assert!(
!matches!(tree.root().expr, NativeExpr::Merge { .. }),
"the root stays `{name}`: {tree}"
);
};
// `timestamp(x)` over the scan, `rate(x[5m])`, and `timestamp(rate(x[5m]))` over an evaluated
// input: the three nodes that drop the metadata labels themselves.
let stamps = || selection(raw_input(0), Arc::clone(&out), InstantValue::Timestamp);
let drops = || window_over(raw_input(0), None, Arc::clone(&out));
let evaluated_stamps = || NativeNode {
expr: NativeExpr::Timestamp {
input: Box::new(drops()),
},
output: evaluated(fields(1), Arc::clone(&out)),
};
// The labels hold: one metric names every series, and no metadata tag is among the tags.
kept("timestamp", stamps());
kept("rate", drops());
kept("timestamp(rate)", evaluated_stamps());
// A source that may hand over several metrics is not proved: the name it drops is what tells
// two of them apart.
collected("timestamp", &named, &[None], stamps());
collected("rate", &named, &[None], drops());
collected("timestamp(rate)", &named, &[None], evaluated_stamps());
// A binding that keys a series by a metadata label is collected: that label goes with the name,
// so two series that differ only there meet.
for metadata in ["__type__", "__unit__"] {
let mut typed = named.clone();
typed[0].push(metadata.to_string());
collected("timestamp", &typed, &single, stamps());
collected("rate", &typed, &single, drops());
collected("timestamp(rate)", &typed, &single, evaluated_stamps());
}
// A window that keeps the name drops nothing, so nothing is asked of it.
let tree = passed_over(
keeping_window_over(raw_input(0), None, Arc::clone(&out)),
&named,
&single,
);
assert_eq!(0, merges(&tree), "`last_over_time` keeps the name: {tree}");
assert!(
matches!(tree.root().expr, NativeExpr::Window { .. }),
"the root stays the name-keeping window: {tree}"
);
}
#[test]
fn info_collects_the_side_that_aliases() {
let out = grid(0, 1000, 2);
@@ -1494,7 +780,7 @@ fn info_collects_the_side_that_aliases() {
}
#[test]
fn a_scalar_broadcast_collects_its_own_name_drop() {
fn a_scalar_side_of_a_broadcast_is_not_collected() {
let out = grid(0, 1000, 2);
let tree = passed(
binary_over(
@@ -1504,23 +790,7 @@ fn a_scalar_broadcast_collects_its_own_name_drop() {
),
&scan_tags(),
);
// `x + 2` drops the metadata labels, so what it hands out is collected - above the match, not
// beside it - and neither operand is collected on its own.
assert_eq!(1, merges(&tree), "{tree}");
let NativeExpr::Merge { input } = &tree.root().expr else {
panic!("the root collects what the broadcast hands out: {tree}");
};
let NativeExpr::Binary { left, right, .. } = &input.expr else {
panic!("the root stays the broadcast");
};
assert!(
matches!(left.expr, NativeExpr::Scalar(_)),
"a constant holds no label set"
);
assert!(
!matches!(right.expr, NativeExpr::Merge { .. }),
"the vector side hands its series over as it reads them"
);
assert_eq!(0, merges(&tree), "a constant holds no label set");
}
#[test]
@@ -1543,7 +813,7 @@ fn aggregate_state_is_not_collected() {
let tree = passed(state, &scan_tags());
assert_eq!(0, merges(&tree));
assert!(matches!(tree.root().output, NativeType::AggregateState(_)));
let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap();
let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap();
assert_eq!(tree.to_string(), twice.to_string());
}
@@ -1567,8 +837,7 @@ fn the_pass_is_idempotent() {
),
&scan_tags(),
);
let twice =
merge_label_rewrites(&once, &scan_tags(), &[], true).expect("a collected tree is legal");
let twice = merge_label_rewrites(&once, &scan_tags()).expect("a collected tree is legal");
assert_eq!(1, merges(&twice));
assert_eq!(once.to_string(), twice.to_string());
}
+341 -207
View File
@@ -18,7 +18,8 @@
//! tree builds independent executions. A caller registers the tree's per-node metric handles into
//! its own metrics set once, before the execution starts.
use std::collections::{HashSet, VecDeque};
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
use common_query::promql_annotations::PromqlAnnotationCollector;
@@ -37,8 +38,8 @@ use datatypes::series::{SeriesLabels, SeriesOrdering};
use crate::aggregate;
use crate::grid_series::{
GridSample, GridSeriesBuilder, fields_bytes, merge_same_labelset, owned, series_bytes,
table_bytes,
GridSample, GridSeriesBuilder, LabelSet, fields_bytes, label_set, merge_same_labelset, owned,
series_bytes, table_bytes,
};
use crate::histogram_eval::float_histogram::FloatHistogram;
use crate::histogram_eval::{
@@ -63,6 +64,9 @@ use crate::native::plan::{
BoundInstant, BoundScalar, BoundSetOp, BoundVectorOp, Grouping, InstantValue, MapOp,
NativeExpr, NativeNode, NativeTree, NativeType, ParameterizedOp, unifies_its_own_contributions,
};
#[cfg(test)]
pub(crate) use crate::native::runs::absorb_rebuilds;
use crate::native::runs::{Run, absorb};
use crate::native::state::{StateChunk, state_bytes};
use crate::range_eval::RangeEvaluator;
use crate::select::{self, SortOrder};
@@ -536,15 +540,6 @@ impl MergeInput {
pub(super) struct NodeBuffer {
metrics: Arc<NodeMetrics>,
reservation: MemoryReservation,
/// What this node keeps for the grid timestamps it materialized, folded into every
/// re-estimate of its own figure; see [`NodeBuffer::reserve_grid`].
///
/// `None` means this node has not reserved a grid yet and `Some(bytes)` that it has: what the
/// record says is that the reservation was made, which is not the same statement as the figure
/// it ended at. A grid always has at least one point today -- `GridSpec` refuses a length of
/// zero -- so the two happen to coincide, but a reservation of no bytes would still be a
/// reservation made, and reading the figure would make it one made per call.
grid: Option<usize>,
}
impl NodeBuffer {
@@ -561,46 +556,9 @@ impl NodeBuffer {
Self {
metrics,
reservation: consumer.register(pool),
grid: None,
}
}
/// Reserves the timestamps of `grid` before this node reads them, once for the execution.
///
/// An [`EvalGrid`] allocates its timestamps on first access and keeps them for as long as the
/// plan holds the grid, which outlives the chunk a node read them for. The node that triggers
/// that allocation therefore reserves the array and keeps the reservation: giving it back with
/// the workspace would leave the execution holding bytes no reservation covers, which is what a
/// grid of a million deferred points -- the reason the array is deferred at all -- would go
/// unnoticed through. Only the node that materializes it reserves it: a grid that is already
/// materialized, whether the plan built it eagerly or a node below or beside this one needed it
/// first, was charged to whoever materialized it, so this node grows by nothing.
///
/// The array is reserved once per node, not once per call: a node is handed its input one chunk
/// at a time and reaches the grid on every chunk it is handed, and a collecting window reaches
/// it again when it flushes what it collected -- which is the call that reads the timestamps,
/// because until then nothing on this node looks at them. Reserving again on each of those
/// calls charges the pool one array per chunk for the one array the execution holds, so a series
/// arriving in `n` pieces would need `n + 1` arrays' worth of budget and exhaust a pool sized
/// for the query over a chunk count that says nothing about what the node keeps. What short
/// circuits the later calls is therefore the reservation this node made (`grid`), not the array
/// having been materialized: it cannot stand in for it, being false exactly while a collecting
/// node is charging for it.
pub(super) fn reserve_grid(&mut self, grid: &EvalGrid) -> Result<()> {
if self.grid.is_some() || grid.is_materialized() {
return Ok(());
}
let len = usize::try_from(grid.spec().len).map_err(|_| {
DataFusionError::Execution("grid length does not fit memory size".to_string())
})?;
let bytes = len.checked_mul(std::mem::size_of::<i64>()).ok_or_else(|| {
DataFusionError::Execution("grid timestamp size overflow".to_string())
})?;
self.grow(bytes)?;
self.grid = Some(bytes);
Ok(())
}
/// Reserves `bytes` beside what this node already holds.
pub(super) fn grow(&mut self, bytes: usize) -> Result<()> {
self.reservation.try_grow(bytes)?;
@@ -609,12 +567,9 @@ impl NodeBuffer {
}
/// Reserves exactly `bytes`, which is what this node holds from now on: a node that consumes its
/// input chunk by chunk is set from the state that is left, not grown per chunk. What a grid's
/// array takes is not part of that figure: the node keeps it for as long as it lives, so it
/// stays reserved on top of what the caller measured.
/// input chunk by chunk is set from the state that is left, not grown per chunk.
pub(super) fn resize(&mut self, bytes: usize) -> Result<()> {
self.reservation
.try_resize(bytes.saturating_add(self.grid.unwrap_or(0)))?;
self.reservation.try_resize(bytes)?;
self.record();
Ok(())
}
@@ -657,7 +612,51 @@ impl Drop for NodeBuffer {
}
}
/// The bytes a metric name holds: its name and each tag's name and value, one row per label.
/// The positions one node has handed out, per output label set and field.
///
/// A node that rewrites labels -- a `Map` dropping the metric name, a `Window` dropping the name and the
/// empty-valued labels -- can turn two input series into one label set, and the node above would read the
/// two chunks as one series. Recording which positions of each field were already handed out lets a second
/// series onto a set be refused before it is evaluated.
///
/// Occupancy is per field and per position, and a chunk holding no value occupies nothing. A set is never
/// retired: `is_last` closes a source series, not a label set. The table is one node's own.
#[derive(Default)]
struct Emitted {
sets: HashMap<LabelSet, History>,
/// How many run vectors, runs and label payload bytes every entry holds, folded in as the entries
/// change rather than by walking them, so [`Emitted::bytes`] is read in constant time.
counts: Counts,
}
/// What the entries of an [`Emitted`] table hold, as running totals.
#[derive(Default)]
struct Counts {
/// The run vectors, one per field of one entry.
vectors: usize,
/// The occupied runs in every vector.
runs: usize,
/// The payload of every label set: the bytes of its names and values with one row per label.
payload: usize,
}
/// What one node has handed out under one output label set.
#[derive(Default)]
struct History {
/// The occupied runs of each field, in position order and disjoint.
fields: Vec<Vec<Run>>,
}
/// The bytes a label set holds: each label's name and value with the pair that refers to them.
fn label_bytes(labels: &LabelSet) -> usize {
labels
.iter()
.map(|(name, value)| std::mem::size_of::<(Arc<str>, Arc<str>)>() + name.len() + value.len())
.sum()
}
/// The bytes a metric name holds: its name and each tag's name and value, one row per label, which is
/// the convention [`label_bytes`] keeps.
fn metric_bytes(metric: &SeriesLabels) -> usize {
metric.name().map_or(0, |name| name.len())
+ std::mem::size_of_val(metric.tags())
@@ -668,6 +667,137 @@ fn metric_bytes(metric: &SeriesLabels) -> usize {
.sum::<usize>()
}
impl Emitted {
/// Records what `chunk` occupies, rejecting it against what this node already handed out.
///
/// Two values of one field at one position is the only thing refused: a second series that fills
/// only the positions the first one left empty merges into it.
fn occupy(&mut self, chunk: &EvaluatedChunk) -> Result<()> {
let labels = label_set(&chunk.series.metric);
let occupied: Vec<Vec<Run>> = chunk
.series
.fields
.iter()
.enumerate()
.map(|(index, field)| occupied_runs(index, field, &chunk.series.time))
.collect::<Result<_>>()?;
let empty = occupied.iter().all(Vec::is_empty);
if let Some(history) = self.sets.get(&labels) {
for (index, runs) in occupied.iter().enumerate() {
let stored = history.fields.get(index).map_or(&[][..], Vec::as_slice);
for run in runs {
// The stored runs are sorted and disjoint, so only the one holding `run.start`
// or the one just before it can share a position.
let at = stored.partition_point(|stored| stored.end <= run.start);
let previous = stored.get(at).filter(|stored| stored.overlaps(*run));
if let Some(previous) = previous {
return Err(occupied_position(&labels, index, run, previous));
}
}
}
}
if empty {
// A chunk holding no value records nothing and closes nothing: a series that answered
// empty leaves every earlier series on this label set exactly where it was.
return Ok(());
}
let fields = chunk.series.fields.len();
let history = match self.sets.entry(labels) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
self.counts.payload += label_bytes(entry.key());
entry.insert(History::default())
}
};
if history.fields.len() < fields {
self.counts.vectors += fields - history.fields.len();
history.fields.resize_with(fields, Vec::new);
}
for (index, runs) in occupied.into_iter().enumerate() {
let stored = &mut history.fields[index];
let before = stored.len();
absorb(stored, &runs);
self.counts.runs = self.counts.runs - before + stored.len();
}
Ok(())
}
/// An estimate of the bytes this table holds, which the node reserves for.
///
/// Read from running totals, so it is constant time; the map's buckets and a vector's slack are
/// left out.
fn bytes(&self) -> usize {
let slots =
self.sets.len() * (std::mem::size_of::<LabelSet>() + std::mem::size_of::<History>());
slots
+ self.counts.vectors * std::mem::size_of::<Vec<Run>>()
+ self.counts.runs * std::mem::size_of::<Run>()
+ self.counts.payload
}
}
/// The runs of positions holding a value in field `index`, as the axis `time` places its rows.
///
/// A missing sample holds no value and a hole between rows is not a position, so a run ends where a hole
/// begins. Every field must hold one sample per row the axis places.
fn occupied_runs(index: usize, field: &Samples, time: &TimeAxis) -> Result<Vec<Run>> {
let mut runs: Vec<Run> = Vec::new();
let mut open: Option<usize> = None;
let mut previous: Option<usize> = None;
let mut cursor = field.cursor();
for (position, row) in time.positions() {
// The axis names the absolute position of every row, so the hole between two rows is
// visible as a jump in `position` while no row of this chunk sits in it.
if let Some(previous) = previous
&& position != previous + 1
&& let Some(start) = open.take()
{
runs.push(Run {
start,
end: previous + 1,
});
}
let sample = cursor
.at(row)
.ok_or_else(|| misaligned_field(index, time, field))?;
let present = !matches!(sample, SampleRef::Missing);
match (present, open) {
(true, None) => open = Some(position),
(false, Some(start)) => {
runs.push(Run {
start,
end: position,
});
open = None;
}
_ => {}
}
previous = Some(position);
}
if let Some(start) = open {
// A run still open at the last row ends just past it, not where the field's row count says:
// the rows need not be adjacent to the positions above them.
runs.push(Run {
start,
end: previous.map_or(start, |last| last + 1),
});
}
Ok(runs)
}
/// The rejection of a series whose axis and fields disagree on how many rows there are.
///
/// [`Series`] keeps its fields public, so a struct-literal chunk can carry fewer samples than the
/// axis places rows.
fn misaligned_field(index: usize, time: &TimeAxis, field: &Samples) -> DataFusionError {
DataFusionError::Execution(format!(
"a series was handed to the node whose axis and fields are misaligned: the axis places {} \
rows, but field {index} holds {} samples, so the axis names a row the field does not have",
time.len(),
field.len(),
))
}
/// What a map node applies.
struct MapRuntime {
ops: Arc<[MapOp]>,
@@ -2064,11 +2194,9 @@ impl BroadcastRuntime {
/// Whether the operation rewrites the label set of what it hands out.
///
/// Arithmetic and a `bool` comparison drop the metric name, so two input series can become one
/// label set under this node; a filtering comparison keeps the label set it read. The answer is
/// read off [`Self::op`], so an operation and its label policy cannot disagree. Collection is not
/// this node's work: a collision the rewritten set creates is what a canonical
/// [`NativeExpr::Merge`] above it refuses.
/// Arithmetic and a `bool` comparison drop the metric name, so two input series can become one label
/// set under this node; a filtering comparison keeps the label set it read. The answer is read off
/// [`Self::op`], so an operation and its label policy cannot disagree.
fn drops_labels(&self) -> bool {
match self.op {
BroadcastOp::Arithmetic(_) | BroadcastOp::Compare(Comparison::Bool(_)) => true,
@@ -2666,10 +2794,11 @@ enum RuntimeOp {
/// State shared by the stateful operators.
///
/// The queue must precede the buffer so it drops before its reservation.
/// The queue and emitted history must precede the buffer so they drop before its reservation.
struct NodeState {
queued: VecDeque<(NativeChunk, usize)>,
queued_tables: usize,
emitted: Option<Emitted>,
buffer: NodeBuffer,
}
@@ -2763,6 +2892,7 @@ impl NodeState {
Self {
queued: VecDeque::new(),
queued_tables: 0,
emitted: None,
buffer: NodeBuffer::new(name, metrics, pool),
}
}
@@ -2789,7 +2919,6 @@ impl RuntimeNode {
binary_before: &mut usize,
location: String,
duplicate_evidence: Option<Arc<DuplicateEvidence>>,
sources: &SourceFacts,
) -> Result<Self> {
stacker::maybe_grow(NATIVE_STACK_RED_ZONE, NATIVE_STACK_GROWTH, || {
Self::build_within_stack(
@@ -2799,7 +2928,6 @@ impl RuntimeNode {
binary_before,
location,
duplicate_evidence,
sources,
)
})
}
@@ -2811,7 +2939,6 @@ impl RuntimeNode {
binary_before: &mut usize,
location: String,
duplicate_evidence: Option<Arc<DuplicateEvidence>>,
sources: &SourceFacts,
) -> Result<Self> {
let metrics = Arc::new(NodeMetrics::default());
let op = match &node.expr {
@@ -2858,7 +2985,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
RuntimeOp::ScalarOf {
input: Box::new(child),
@@ -2905,7 +3031,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
RuntimeOp::VectorOf {
input: Box::new(child),
@@ -2927,7 +3052,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let input_schema = input.output.samples().cloned().ok_or_else(|| {
DataFusionError::Execution(
@@ -2938,11 +3062,15 @@ impl RuntimeNode {
input: Box::new(child),
instant: InstantRuntime::try_new(&input_schema, op)?,
state: {
NodeState::new(
let mut state = NodeState::new(
format!("native.instant at {location}"),
Arc::clone(&metrics),
pool,
)
);
if op.value == InstantValue::Timestamp {
state.emitted = Some(Emitted::default());
}
state
},
}
}
@@ -2954,7 +3082,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let schema = node.output.samples().cloned().ok_or_else(|| {
DataFusionError::Execution("merge does not produce samples".to_string())
@@ -2978,23 +3105,18 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let input_schema = input.output.samples().cloned().ok_or_else(|| {
DataFusionError::Execution("window consumes non-sample input".to_string())
})?;
// The child reports fragments only when it cannot prove the labels it drops tell its
// series apart, which is the same question the canonical pass asked before it
// decided this window needs no merge above it.
let unification =
unifies_its_own_contributions(input, &sources.tags, &sources.metrics);
let merge = matches!(unification, Some(false))
let merge = matches!(unifies_its_own_contributions(input), Some(false))
.then(|| MergeInput::new(Arc::clone(&input_schema)));
let state = NodeState::new(
let mut state = NodeState::new(
format!("native.window at {location}"),
Arc::clone(&metrics),
pool,
);
state.emitted = Some(Emitted::default());
RuntimeOp::Window {
input: Box::new(child),
// The collector arrives through `set_annotations` before the first pull;
@@ -3017,16 +3139,16 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let schema = node.output.samples().cloned().ok_or_else(|| {
DataFusionError::Execution("map does not produce samples".to_string())
})?;
let state = NodeState::new(
let mut state = NodeState::new(
format!("native.map at {location}"),
Arc::clone(&metrics),
pool,
);
state.emitted = Some(Emitted::default());
RuntimeOp::Map {
input: Box::new(child),
map: MapRuntime {
@@ -3048,7 +3170,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
// A function of two scalars answers a scalar, which has one unlabelled field on
// the grid it is evaluated on; every other function here answers its input's own
@@ -3087,15 +3208,15 @@ impl RuntimeNode {
binary_before,
format!("{location}.parameter{index}"),
duplicate_evidence.clone(),
sources,
)
})
.collect::<Result<Vec<_>>>()?;
let state = NodeState::new(
let mut state = NodeState::new(
format!("native.map at {location}"),
Arc::clone(&metrics),
pool,
);
state.emitted = Some(Emitted::default());
RuntimeOp::ParameterizedMap {
input: Box::new(child),
parameters: nodes,
@@ -3119,16 +3240,16 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let schema = node.output.samples().cloned().ok_or_else(|| {
DataFusionError::Execution("timestamp does not produce samples".to_string())
})?;
let state = NodeState::new(
let mut state = NodeState::new(
format!("native.timestamp at {location}"),
Arc::clone(&metrics),
pool,
);
state.emitted = Some(Emitted::default());
RuntimeOp::Timestamp {
input: Box::new(child),
schema,
@@ -3167,7 +3288,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.left"),
duplicate_evidence.clone(),
sources,
)?;
let right_node = Self::build(
right,
@@ -3176,7 +3296,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.right"),
duplicate_evidence.clone(),
sources,
)?;
match (binary, broadcast) {
(Some(binary), None) => RuntimeOp::Binary {
@@ -3190,11 +3309,18 @@ impl RuntimeNode {
),
},
(None, Some(broadcast)) => {
let state = NodeState::new(
let vector = if broadcast.vector_left { left } else { right };
let mut state = NodeState::new(
format!("native.binary at {location}"),
Arc::clone(&metrics),
pool,
);
// Only an operation that rewrites the label set is where two input series
// can become one, and only one that makes an identity its vector side has
// not already checked needs a table of its own.
if broadcast.drops_labels() && !elides_emitted_table(&broadcast, vector) {
state.emitted = Some(Emitted::default());
}
RuntimeOp::Broadcast {
left: Box::new(left_node),
right: Box::new(right_node),
@@ -3232,7 +3358,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.left"),
duplicate_evidence.clone(),
sources,
)?;
let right_node = Self::build(
right,
@@ -3241,7 +3366,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.right"),
duplicate_evidence.clone(),
sources,
)?;
RuntimeOp::Binary {
left: Box::new(left_node),
@@ -3281,7 +3405,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.base"),
duplicate_evidence.clone(),
sources,
)?;
let info_node = Self::build(
info,
@@ -3290,7 +3413,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.info"),
duplicate_evidence.clone(),
sources,
)?;
RuntimeOp::Binary {
left: Box::new(base_node),
@@ -3325,7 +3447,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
RuntimeOp::Aggregate {
input: Box::new(child),
@@ -3355,7 +3476,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let parameter = Self::build(
parameter,
@@ -3364,7 +3484,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.parameter"),
duplicate_evidence.clone(),
sources,
)?;
let state = NodeState::new(
format!("native.vector at {location}"),
@@ -3419,7 +3538,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let mut built = Vec::with_capacity(parameters.len());
for (index, parameter) in parameters.iter().enumerate() {
@@ -3430,7 +3548,6 @@ impl RuntimeNode {
binary_before,
format!("{location}.parameter{index}"),
duplicate_evidence.clone(),
sources,
)?;
built.push(Some(runtime));
}
@@ -3474,15 +3591,9 @@ impl RuntimeNode {
binary_before,
format!("{location}.input"),
duplicate_evidence.clone(),
sources,
)?;
let limitk = matches!(op, BoundVectorOp::LimitK { .. });
if limitk
&& matches!(
unifies_its_own_contributions(input, &sources.tags, &sources.metrics),
Some(false)
)
{
if limitk && matches!(unifies_its_own_contributions(input), Some(false)) {
vector.merge_input = true;
}
RuntimeOp::Vector {
@@ -3839,12 +3950,6 @@ impl RuntimeNode {
state,
} => {
if !std::mem::replace(produced, true) {
// `time()` and the date functions read every point of the grid, so this node is
// what materializes a deferred grid's timestamps.
state
.buffer
.reserve_grid(grid)
.map_err(|error| state.context(error))?;
let schema = scalar_schema(Arc::clone(grid));
state
.produce(grid, || match date {
@@ -4268,7 +4373,7 @@ impl RuntimeNode {
.settle_vector(operation)
.map_err(|error| state.context(error))?;
// Finish windows before dropping the consumed subtree; its children may still
// hold charged buffers or scalar series even after the scalar was read.
// hold charged emitted history or scalar series even after the scalar was read.
parameter_node.finish_windows()?;
*parameter = None;
}
@@ -4801,13 +4906,9 @@ impl RuntimeNode {
..
} => {
let child = input.finish_windows();
// Finishing evaluates the points still owed, which reads the grid's timestamps.
let own = window
.take()
.map_or(Ok(()), |evaluator| {
state.buffer.reserve_grid(evaluator.grid())?;
RangeEvaluator::finish(evaluator)
})
.map_or(Ok(()), RangeEvaluator::finish)
.map_err(|error| state.context(error));
child.and(own)
}
@@ -4815,59 +4916,11 @@ impl RuntimeNode {
}
}
/// What a fused execution knows about the labels its inputs hand over.
///
/// The planner binds the tree's inputs and knows two things about each: the complete label set it
/// reports, and - when the statement reads one table - the one metric every series it hands over is
/// named. [`unifies_its_own_contributions`] reads exactly those facts, so the collection a consumer
/// performs at run time is decided by the same proof the canonical pass placed its merges with. A
/// caller that knows nothing sets no facts, and every consumer then collects conservatively.
#[derive(Clone, Debug, Default)]
pub struct SourceFacts {
/// The complete runtime label set of each input, indexed by the `source_index` of its leaf.
tags: Vec<Vec<String>>,
/// The one metric each input is bound to, indexed the same way; `None` when the input may hand
/// over several, which is what a statement reading several tables binds.
metrics: Vec<Option<Arc<str>>>,
}
impl SourceFacts {
/// Says nothing about any input, so every decision stays conservative.
pub fn none() -> Self {
Self::default()
}
/// One statement-level fact, repeated for each of `sources` inputs.
///
/// A binding names one metric and one tag set for the node as a whole, and every scan below it
/// reads that one table, so the fact holds for each of them. A binding that names no metric -
/// the node reads one raw source per input - proves nothing, and the proof fails with it.
pub fn uniform(tags: Vec<String>, metric: Option<Arc<str>>, sources: usize) -> Self {
Self {
tags: vec![tags; sources],
metrics: vec![metric; sources],
}
}
}
impl NativeExecutionState {
pub fn try_new(
tree: &NativeTree,
pool: Arc<dyn MemoryPool>,
duplicate_evidence: Option<Arc<DuplicateEvidence>>,
) -> Result<Self> {
Self::try_new_with_sources(tree, pool, duplicate_evidence, &SourceFacts::none())
}
/// Builds the execution over `tree`, judging a dropped label set by `sources`.
///
/// The canonical pass placed its merges with the same facts, so a consumer that collects here
/// collects exactly where the pass decided a consumer has to - and nowhere else.
pub fn try_new_with_sources(
tree: &NativeTree,
pool: Arc<dyn MemoryPool>,
duplicate_evidence: Option<Arc<DuplicateEvidence>>,
sources: &SourceFacts,
) -> Result<Self> {
let mut inputs = Vec::new();
let mut binary_before = 0usize;
@@ -4878,7 +4931,6 @@ impl NativeExecutionState {
&mut binary_before,
"root".to_string(),
duplicate_evidence.clone(),
sources,
)?;
// A constant root has no node that produces its value, so the state produces it.
let scalar_root = match (&tree.root().expr, &tree.root().output) {
@@ -5092,18 +5144,21 @@ impl NodeState {
let output = match outcome {
Ok(output) => output,
Err(error) => {
self.buffer.resize(0)?;
self.buffer.resize(self.emitted_bytes())?;
return Err(error);
}
};
if let Some(output) = &output {
metrics.record_grid_points(output.series.time.len());
metrics.record_output(1, valid_values(output));
// What it handed out is what the label sets of this node now hold: `map_chunk` has
// already rewritten the output's label set, so the label set of the output is the one
// two input series can share.
self.track(output)?;
}
// The chunk is gone and this node holds none of it: a rewritten label set is not this
// node's to remember, and what two series sharing one would mean is a canonical merge's
// question.
self.buffer.resize(0)?;
// The chunk is gone and the label sets this node has handed out are what is left.
let emitted = self.emitted_bytes();
self.buffer.resize(emitted)?;
Ok(output)
}
@@ -5133,7 +5188,8 @@ impl NodeState {
let outcome = match outcome {
Ok(outcome) => outcome,
Err(error) => {
self.buffer.resize(map.parameter_bytes)?;
self.buffer
.resize(self.emitted_bytes().saturating_add(map.parameter_bytes))?;
return Err(error);
}
};
@@ -5151,9 +5207,11 @@ impl NodeState {
if let Some(output) = &output {
metrics.record_grid_points(output.series.time.len());
metrics.record_output(1, valid_values(output));
self.track(output)?;
}
// What this node still holds is what it read for its parameters and nothing else.
self.buffer.resize(map.parameter_bytes)?;
let emitted = self.emitted_bytes();
self.buffer
.resize(emitted.saturating_add(map.parameter_bytes))?;
Ok(output)
}
@@ -5165,16 +5223,21 @@ impl NodeState {
) -> Result<Option<EvaluatedChunk>> {
let positions = chunk.series.time.len();
let closing = chunk.is_last;
// The kernel reads the grid's timestamps, which materializes them when the grid is
// deferred. They stay materialized for the rest of the execution, so this node reserves
// them beside what it holds and keeps them there rather than treating them as a workspace
// it gives back.
if let Some(grid) = schema.eval_grid.as_deref() {
self.buffer.reserve_grid(grid)?;
}
// The kernel reads the grid's timestamps, which it materializes when they are deferred.
let grid_points = schema.eval_grid.as_ref().map_or(0, |grid| {
usize::try_from(grid.spec().len).unwrap_or(usize::MAX)
});
let workspace = built_series_workspace_bytes(&chunk.series)?;
let scratch = grid_points
.checked_mul(std::mem::size_of::<i64>())
.and_then(|timestamps| timestamps.checked_add(workspace))
.ok_or_else(|| {
DataFusionError::Execution(
"the workspace of timestamp does not fit one address space".to_string(),
)
})?;
self.buffer.grow(evaluated_bytes(&chunk))?;
self.buffer.grow(workspace)?;
self.buffer.grow(scratch)?;
let labels = drop_metric_name(&chunk.series.metric)?;
let metrics = Arc::clone(&self.buffer.metrics);
let result = {
@@ -5189,7 +5252,7 @@ impl NodeState {
let result = match result {
Ok(result) => result,
Err(error) => {
self.buffer.resize(0)?;
self.buffer.resize(self.emitted_bytes())?;
return Err(error);
}
};
@@ -5201,8 +5264,9 @@ impl NodeState {
if let Some(output) = &output {
metrics.record_grid_points(output.series.time.len());
metrics.record_output(1, valid_values(output));
self.track(output)?;
}
self.buffer.resize(0)?;
self.buffer.resize(self.emitted_bytes())?;
Ok(output)
}
@@ -5249,7 +5313,8 @@ impl NodeState {
let output = match outcome {
Ok(output) => output,
Err(error) => {
self.buffer.resize(broadcast.scalar_bytes)?;
self.buffer
.resize(self.emitted_bytes().saturating_add(broadcast.scalar_bytes))?;
return Err(error);
}
};
@@ -5260,9 +5325,17 @@ impl NodeState {
if let Some(output) = &output {
metrics.record_grid_points(output.series.time.len());
metrics.record_output(1, valid_values(output));
// Only an operation that rewrites the label set is where two input series can become one, and
// only that one keeps the table `track` records into.
if broadcast.drops_labels() {
self.track(output)?;
}
}
// The chunk is gone and the scalar's value is what is left.
self.buffer.resize(broadcast.scalar_bytes)?;
// The chunk is gone: the label sets this node has handed out and the scalar's value are
// what is left.
let emitted = self.emitted_bytes();
self.buffer
.resize(emitted.saturating_add(broadcast.scalar_bytes))?;
Ok(output)
}
@@ -5328,9 +5401,6 @@ impl NodeState {
merge: &mut Option<MergeInput>,
chunk: NativeChunk,
) -> Result<Option<EvaluatedChunk>> {
// A window reads the grid's timestamps to place every point it decides, which materializes
// a deferred grid; the array stays materialized, so the reservation for it is kept.
self.buffer.reserve_grid(window.grid())?;
if let Some(merge) = merge.as_mut() {
let NativeChunk::Evaluated(chunk) = chunk else {
return Err(DataFusionError::Execution(
@@ -5391,19 +5461,22 @@ impl NodeState {
let output = match outcome {
Ok(output) => output,
Err(error) => {
self.buffer.resize(window.retained_bytes())?;
self.buffer
.resize(window.retained_bytes().saturating_add(self.emitted_bytes()))?;
return Err(error);
}
};
if let Some(output) = &output {
metrics.record_grid_points(output.series.time.len());
metrics.record_output(1, valid_values(output));
self.track(output)?;
}
// The chunk has been consumed: what this node still holds is what the evaluator kept of it,
// charged whether or not this chunk produced an answer: a chunk that yields nothing leaves
// the evaluator's samples where they were, and the figure is not charged per chunk consumed,
// which would make a fixed window over a long series read as an allocation failure.
self.buffer.resize(window.retained_bytes())?;
// The chunk has been consumed: what this node still holds is what the evaluator kept of it and the
// label sets it has already handed out. Neither is charged per chunk consumed, which would make a
// fixed window over a long series read as an allocation failure.
let retained = window.retained_bytes();
let emitted = self.emitted_bytes();
self.buffer.resize(retained.saturating_add(emitted))?;
Ok(output)
}
@@ -5693,11 +5766,6 @@ impl NodeState {
));
};
let (positions, closing) = (chunk.time.len(), chunk.is_last);
// A selection that answers with timestamps reads every point of the grid, which materializes
// a deferred grid's array; the reservation is kept because the array is.
if instant.selection.value != InstantValue::Sample {
self.buffer.reserve_grid(&instant.selection.output_grid)?;
}
// The chunk is held from here on: a selection keeps every chunk of a series until the one
// that closes it, so what this node reserves for is the series it has buffered.
self.buffer.grow(raw_bytes(&chunk))?;
@@ -5736,6 +5804,9 @@ impl NodeState {
if let Some(output) = &output {
metrics.record_grid_points(output.series.time.len());
metrics.record_output(1, valid_values(output));
if instant.selection.value == InstantValue::Timestamp {
self.track(output)?;
}
}
// What this node still holds is the chunks of the series it is selecting: a whole series is held
// until it ends. The chunk just consumed and the gather's workspace are both gone, so this gives
@@ -5746,7 +5817,11 @@ impl NodeState {
/// Sets this node's reservation to the chunks of the series it is still gathering.
fn settle_buffered(&mut self, instant: &InstantRuntime) -> Result<()> {
self.buffer.resize(instant.buffered_bytes())
self.buffer.resize(
instant
.buffered_bytes()
.saturating_add(self.emitted_bytes()),
)
}
/// Retains and validates a fragment for canonicalization.
@@ -5803,9 +5878,12 @@ impl NodeState {
chunk: EvaluatedChunk,
) -> Result<Option<EvaluatedChunk>> {
self.collect_merge(merge, chunk)?;
// What this node holds is the fragments; its evaluator has read none of them yet.
// What this node holds is the fragments and the label sets it has already handed out; its
// evaluator has read none of them yet.
let retained = window.retained_bytes();
self.buffer.resize(retained.saturating_add(merge.bytes))?;
let emitted = self.emitted_bytes();
self.buffer
.resize(retained.saturating_add(emitted).saturating_add(merge.bytes))?;
Ok(None)
}
@@ -5827,8 +5905,6 @@ impl NodeState {
// evaluator never read a series and the charge goes back.
return self.buffer.shrink(merge.bytes);
}
// The merged series is evaluated by the window, which reads the grid's timestamps.
self.buffer.reserve_grid(window.grid())?;
// The workspace the merge allocates is charged before it runs, from the rows, segments and
// fields of the fragments as they were collected.
let workspace = merge.workspace_bytes();
@@ -5869,9 +5945,15 @@ impl NodeState {
.metrics
.record_grid_points(output.series.time.len());
self.buffer.metrics.record_output(1, valid_values(&output));
self.track(&output)?;
let retained = window.retained_bytes();
self.buffer
.resize(retained.saturating_add(held).saturating_add(queued_bytes))?;
let emitted = self.emitted_bytes();
self.buffer.resize(
retained
.saturating_add(emitted)
.saturating_add(held)
.saturating_add(queued_bytes),
)?;
let bytes = evaluated_bytes(&output);
self.enqueue_output(output)?;
queued_bytes = queued_bytes.saturating_add(bytes);
@@ -5909,6 +5991,26 @@ impl NodeState {
}
Ok(())
}
/// Records what this node hands out under its output label set.
///
/// A node that rewrites labels is where two input series become indistinguishable, so this is where a
/// second one is refused: checking at the root would be too late for a window above, which would
/// already have evaluated the two as two series.
fn track(&mut self, output: &EvaluatedChunk) -> Result<()> {
match self.emitted.as_mut() {
Some(emitted) => emitted.occupy(output),
// A node that keeps its input's labels cannot make two series one.
None => Ok(()),
}
}
/// The bytes the label sets this node has handed out hold.
///
/// Zero for a node that keeps its input's labels, which keeps no such table.
fn emitted_bytes(&self) -> usize {
self.emitted.as_ref().map_or(0, Emitted::bytes)
}
}
impl InputSlot {
@@ -6366,6 +6468,24 @@ fn missing_row(index: usize) -> DataFusionError {
))
}
/// The rejection of a chunk holding a value where the same field already holds one.
///
/// No merge resolves this one: at `position`, field `field` of the label set `labels` would carry two
/// values, which is what Prometheus rejects for two series whose label sets became equal.
fn occupied_position(
labels: &LabelSet,
field: usize,
run: &Run,
previous: &Run,
) -> DataFusionError {
DataFusionError::Execution(format!(
"vector cannot contain metrics with the same labelset {}: two series hold a value of field \
{field} at position {position} ({previous:?} and {run:?})",
crate::grid_series::render_label_set(labels),
position = run.start.max(previous.start),
))
}
/// The rejection of an input that ended without the chunk closing its series.
///
/// A tree with no window has no evaluator to hold the series, so the input node is the one node left that
@@ -6499,6 +6619,20 @@ fn broadcast_runtime(
})
}
/// Whether the [`Emitted`] table `broadcast` would keep only repeats the one its vector side keeps.
///
/// Literal arithmetic leaves missing samples missing and floats at their positions, so its output occupancy
/// is a subset of the window's, and the second label drop removes only empty-valued labels that
/// [`label_set`] ignores. Arithmetic therefore cannot create a collision the window did not already refuse.
fn elides_emitted_table(broadcast: &BroadcastRuntime, vector: &NativeNode) -> bool {
matches!(broadcast.scalar, BroadcastScalar::Literal(_))
&& matches!(broadcast.op, BroadcastOp::Arithmetic(_))
&& matches!(
&vector.expr,
NativeExpr::Window { op, .. } if op.function.drops_metric_name()
)
}
/// The operation a binary node's `matching` describes.
///
/// A comparison is a filter unless it was bound as `bool`, and a filter keeps the label set it read while
File diff suppressed because it is too large Load Diff
+1
View File
@@ -34,6 +34,7 @@ pub mod display;
pub mod execute;
pub mod metrics;
pub mod plan;
pub(crate) mod runs;
pub mod segment;
pub(crate) mod split;
pub(crate) mod state;
+39 -519
View File
@@ -1307,10 +1307,9 @@ impl NativeNode {
/// A canonical label set is the identity a [`NativeExpr::Merge`] groups by: the metric name plus the
/// ordinary labels, which is exactly what [`SeriesLabels`] holds. One scan partition holds one
/// series per such set - a stored series is one identity, and the scan splits rows by storage key
/// rather than by label set - so only a computation that *writes* an existing label, or drops one,
/// can turn two of those identities into one. A `label_replace` or a `label_join` writes one, and it
/// may name the very label that told its input's series apart; a range function that drops the
/// metric name, and `timestamp` in either of its forms, drop one.
/// rather than by label set - so only a computation that *writes* an existing label can turn two of
/// those identities into one. A `label_replace` or `label_join` does exactly that, and it may name
/// the very label that told its input's series apart.
///
/// The rewrite is judged by the labels it writes and the ones its input carries, which is why the
/// caller supplies what it knows about each source's tags, indexed by the `source_index` the leaves
@@ -1332,47 +1331,23 @@ impl NativeNode {
///
/// Everything else either keeps the set it read or answers a set of its own:
///
/// - A map of math drops the metadata labels, and so do a histogram map, a parameterized map, an
/// arithmetic or `bool` scalar broadcast, a range function that computes its result and a
/// `timestamp` in either of its forms. Such a node is judged by two facts
/// ([`metadata_drop_keeps_series_apart`]): the labels it and its input report, and the scan's own
/// key columns. A source bound to one metric names every series it hands over, so the name the
/// drop removes told none of them apart, and a source that may hand over several is collected; and
/// because the drop removes every empty label value with the name, a key column that holds `NULL`
/// in one series and `''` in another - an absent and an empty value, which is one label set once
/// the empty one is gone - is collected too. `keys_survive` is that second fact, stated by the
/// caller for the scan below the tree
/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), which is
/// the same rule the physical per-series scan applies); a caller that cannot state it passes
/// `false`, and the conservative answer is the one that collects.
/// - A map of math drops the metric name, and an arithmetic broadcast does too, but within one
/// metric the remaining tags are the primary key: two stored series of one metric that agree on
/// every other label are the same series, so dropping the name cannot make them one. Each of
/// those nodes hands its input's answer on rather than claiming one of its own.
/// - An `Aggregate` answers its groups, a `Vector` that folds series answers its result sets, and a
/// `Binary` that reads whole vectors files its rows by the label set a sample *ends* under and
/// refuses two values of a field at one point. A `Set` does the same, and `info` merges the
/// results that share a label set. Each answers one series per set it names.
/// - A `Window` that drops the metadata labels, and the two timestamps, claim one of their own: the
/// name is part of the identity a merge groups by, so two series that shared every other label are
/// one label set from there on - unless the drop tells them apart by the same two-fact rule the
/// kernels above are judged by.
/// - The empty-value half of that proof is answered by a [`Merge`](NativeExpr::Merge) at or below the
/// drop ([`canonical_below`]): a merge hands over one series per canonical label set - the
/// comparison that treats an empty label value as no label - so no drop above it can fold two of
/// its series by removing an empty value. What a merge does not answer is the metadata labels
/// themselves, which the labels half below still has to prove. That is what keeps a chain of drops
/// over one metric to the one collection at its bottom rather than one per drop.
/// - A node that hands its input's series over unchanged - a selection, a name-keeping window,
/// `topk` and its ranking siblings - passes the answer on: an alias below it is still an alias
/// above it.
/// - A node that hands its input's series over unchanged - a selection, a window, a timestamp, a
/// parameterized map, `topk` and its ranking siblings - passes the answer on: an alias below it is
/// still an alias above it.
///
/// This is the fact [`merge_label_rewrites`](crate::native::canonicalize::merge_label_rewrites)
/// inserts a `Merge` for: a consumer that reads a whole label set cannot answer from pieces, and this
/// says where pieces of one set can exist at all. A `Merge` is the collection itself, so it answers
/// `false` - which is what makes the pass idempotent.
pub fn duplicates_canonical_labels(
node: &NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> bool {
pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec<String>]) -> bool {
match &node.expr {
NativeExpr::Input(_)
| NativeExpr::Scalar(_)
@@ -1380,50 +1355,13 @@ pub fn duplicates_canonical_labels(
| NativeExpr::VectorOf { .. } => false,
// The collection itself: one series per label set comes out of it.
NativeExpr::Merge { .. } => false,
// These hand each series over as they read it, so what aliases below them still aliases --
// except `timestamp`, which reports a sample's own time under the metadata labels dropped
// and is therefore a source of duplicates in its own right. A merge below it already hands
// over one series per label set, so nothing the drop can fold is left there.
NativeExpr::Instant { input, op } => {
(op.value == InstantValue::Timestamp
&& !metadata_drop_keeps_series_apart(
input,
source_tags,
source_metrics,
keys_survive,
))
|| duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
}
// A window that drops the metadata labels can fold two series that agreed on every other
// label into one set, so it is a source of duplicates whether or not its input was one -
// unless the labels and the scan's key columns provably tell those series apart still, or a
// merge below it already collected them.
NativeExpr::Window { input, op } => {
(op.function.drops_metric_name()
&& !metadata_drop_keeps_series_apart(
input,
source_tags,
source_metrics,
keys_survive,
))
|| duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
}
// `timestamp` always drops the metadata labels, so two series it reads can become one set -
// unless the labels below it are all bound to one metric and the scan's keys survive the
// drop.
NativeExpr::Timestamp { input } => {
!metadata_drop_keeps_series_apart(input, source_tags, source_metrics, keys_survive)
|| duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
}
// A parameterized map is a math kernel - `round`, `clamp` - and drops the metadata labels as
// a math map does, so it answers for the same reason. Its parameter is a scalar and carries
// no labels of the input's.
NativeExpr::ParameterizedMap { input, .. } => {
!metadata_drop_keeps_series_apart(input, source_tags, source_metrics, keys_survive)
|| duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
}
NativeExpr::ParameterizedTopK { input, .. } => {
duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
// These hand each series over as they read it, so what aliases below them still aliases.
NativeExpr::Instant { input, .. }
| NativeExpr::Window { input, .. }
| NativeExpr::Timestamp { input }
| NativeExpr::ParameterizedMap { input, .. }
| NativeExpr::ParameterizedTopK { input, .. } => {
duplicates_canonical_labels(input, source_tags)
}
// `scalar()` answers one value, which names no label set at all.
NativeExpr::ScalarOf { .. } => false,
@@ -1435,21 +1373,9 @@ pub fn duplicates_canonical_labels(
// leaves every series' identity distinct. `map_writes_existing_label` is that question, and
// an input whose labels this walk cannot derive is collected rather than trusted. Even a
// whole-vector consumer can emit distinct series whose labels a later rewrite makes equal.
// A math or histogram kernel in the same map drops the metadata labels, which is a source of
// duplicates of its own unless the labels and the scan's keys tell its input's series apart -
// see the doc comment above.
NativeExpr::Map { input, op } => {
(op.ops
.iter()
.any(|operation| matches!(operation, MapOp::Math(_) | MapOp::Histogram(_)))
&& !metadata_drop_keeps_series_apart(
input,
source_tags,
source_metrics,
keys_survive,
))
|| map_writes_existing_label(input, op, source_tags).unwrap_or(true)
|| duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
map_writes_existing_label(input, op, source_tags).unwrap_or(true)
|| duplicates_canonical_labels(input, source_tags)
}
// A fold answers one series per group, which is one label set per group.
NativeExpr::Aggregate { .. } => false,
@@ -1458,49 +1384,23 @@ pub fn duplicates_canonical_labels(
BoundVectorOp::TopK { .. }
| BoundVectorOp::LimitK { .. }
| BoundVectorOp::LimitRatio { .. }
| BoundVectorOp::Sort(_) => {
duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive)
}
| BoundVectorOp::Sort(_) => duplicates_canonical_labels(input, source_tags),
// `absent` reports one series, `count_values` one per distinct value, and the histogram
// functions fold the bucket series of one histogram into one series.
BoundVectorOp::Absent { .. }
| BoundVectorOp::Histogram(_)
| BoundVectorOp::CountValues { .. } => false,
},
NativeExpr::Binary {
left,
right,
matching,
} => {
NativeExpr::Binary { left, right, .. } => {
let vector = |side: &NativeNode| side.output.samples().is_some();
match (vector(left), vector(right)) {
if vector(left) && vector(right) {
// Two vectors are matched by label, and the kernel files its rows by the label set a
// sample ends under, refusing two values of one field at one point.
(true, true) => false,
// A broadcast reaches each series of its vector side as it is. An arithmetic one and
// one that reports a `bool` drop the metadata labels, which is a source of
// duplicates of its own unless the labels and the scan's keys tell its series apart;
// a filtering comparison keeps the set it read. A merge below the vector side already
// handed over one series per label set, so the drop has nothing left to fold there.
(true, false) | (false, true) => {
let carried = if vector(left) { left } else { right };
let drops_metadata = !matching.op.is_comparison() || matching.return_bool;
(drops_metadata
&& !metadata_drop_keeps_series_apart(
carried,
source_tags,
source_metrics,
keys_survive,
))
|| duplicates_canonical_labels(
carried,
source_tags,
source_metrics,
keys_survive,
)
}
// Two scalars name no label set at all.
(false, false) => false,
false
} else {
// A broadcast reaches each series of its vector side as it is.
let carried = if vector(left) { left } else { right };
duplicates_canonical_labels(carried, source_tags)
}
}
// A set operation keeps each side's samples and files them by result label set, refusing two
@@ -1547,313 +1447,6 @@ fn map_writes_existing_label(
}
Some(existing)
}
/// Whether the metadata labels a rewrite drops are the only ones telling `node`'s series apart.
///
/// A range function that computes its result and `timestamp` in either of its forms drop
/// [`PROMQL_METADATA_LABELS`] - the metric name, `__type__` and `__unit__` - and a consumer above
/// them that reads a whole label set has to establish those series cannot collide once the three are
/// gone. Two series of one input collide only if they differed in one of the three alone,
/// so the drop is injective - and nothing has to be collected - when neither can tell two of them
/// apart:
///
/// - The name: a source bound to one metric names every series it hands over, so the name is the
/// same for all of them and removing it merges nothing (see [`single_metric_name`]). A source that
/// may hand over several - one selector that discovered several metrics, or a caller that cannot
/// tell - is not proved, and the conservative answer is the one that collects.
/// - `__type__` and `__unit__`: both are removed along with the name, and neither is a label this
/// walk can prove constant across a source's series. They are ordinary labels when a source
/// carries them as tags or a rewrite writes them, and two series that agree on everything else are
/// one label set once they are gone. Their presence anywhere in `node`'s output is therefore
/// enough to withhold the proof.
///
/// A node whose labels already lost the name - `abs`, a name-dropping window below it - has no name
/// of its own left to remove, and what its surviving labels tell apart is then only as good as the
/// metrics below it: a drop that happened on the way there removed a name without asking this walk,
/// so the proof asks instead that every source it reads is bound to the same one metric
/// ([`single_source_metric`]). Over a selector that discovered several metrics that drop folds them
/// together, and the answer is the one that collects.
///
/// `false` when the walk cannot derive `node`'s labels at all, because an unknown set proves nothing.
///
/// This is the whole of the drop's proof: every arm of [`duplicates_canonical_labels`] that decides
/// whether a collection is needed asks it, and so does [`unifies_its_own_contributions`] at run
/// time. What it cannot see is a key column that holds `NULL` in one series and `''` in another,
/// both of which decode to no label once the empty value is dropped with the name. That fact is not
/// derivable from a tree - it is read from the resolved table - so the caller states it as
/// `keys_survive`, using the same rule the physical per-series scan applies
/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop)).
/// [`leaves_a_drop_to_the_storage_keys`] is what makes the statements collected for that reason
/// countable.
fn metadata_drop_keeps_series_apart(
node: &NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> bool {
// The empty-value half is asked first, because it is the cheap one and because a merge at or
// below `node` answers it for the tree ([`canonical_below`]): a merge hands over one series per
// canonical label set - the comparison that treats an empty label value as no label - so two of
// its series are already distinct under it, and no drop above can fold them by removing an empty
// value. What a merge does *not* answer is the metadata labels themselves: it keeps two series
// that differ in the name, `__type__` or `__unit__`, and those are exactly what the drop
// removes, so the labels half still has to hold on its own.
//
// A caller that cannot state the key half - one holding no table, or a tree whose scans
// disagree - passes `keys_survive = false`, and only a merge below it can then carry the drop.
(keys_survive || canonical_below(node))
&& metadata_labels_keep_series_apart(node, source_tags, source_metrics)
}
/// Whether the labels alone tell two of `node`'s series apart once the metadata labels are dropped.
///
/// This is the half of [`metadata_drop_keeps_series_apart`] a tree can answer about itself; the other
/// half is the empty values the drop removes - the scan's key columns, or a merge that has already
/// told those apart. See that function's doc comment for the rule itself.
fn metadata_labels_keep_series_apart(
node: &NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
) -> bool {
let Some(tags) = fused_output_tags(node, source_tags) else {
return false;
};
if tags
.iter()
.any(|tag| tag == "__type__" || tag == "__unit__")
{
return false;
}
if !tags.iter().any(|tag| tag == METRIC_NAME) {
// No name is left to lose, so what the surviving labels tell apart is decided by the sources
// below: every one of them reading the same single metric is what says a name dropped on the
// way here could not have folded two metrics together either. `abs` and a name-dropping
// window both drop the name without asking, so without that fact nothing below is proved.
return single_source_metric(node, source_metrics).is_some();
}
single_metric_name(node, source_metrics).is_some()
}
/// Whether a [`NativeExpr::Merge`] collects the series `node` hands over, at it or below it.
///
/// A merge holds one series per canonical label set, so the series above one are not stored series
/// any more: a merge with nothing to merge hands its input's pieces through, and one that does
/// merge keeps the first piece's labels and drops the pieces it folded, so what comes out is unique
/// under the label set Prometheus compares (`label_set` in `grid_series`) - the one that treats an
/// empty label value as no label. Two pieces that came out of one merge are therefore distinct under
/// that comparison even though their rows may still spell an empty value, which is what makes a
/// second drop above the merge unable to fold them by removing an empty value. What it does not make
/// impossible is the rest of such a drop: a merge keeps two series that differ in `__name__`,
/// `__type__` or `__unit__`, and removing those is exactly what a metadata drop does.
///
/// Only the children that carry samples are searched: a parameter is a scalar or a vector of
/// constants, and a merge below one reads series this node's own samples did not come from.
fn canonical_below(node: &NativeNode) -> bool {
matches!(node.expr, NativeExpr::Merge { .. })
|| node
.expr
.children()
.into_iter()
.filter(|child| matches!(child.output, NativeType::Samples(_)))
.any(canonical_below)
}
/// Whether `node` drops [`PROMQL_METADATA_LABELS`] on its own over series whose labels are proved to
/// tell apart and whose keys do not.
///
/// This is the question of what the drop costs: the labels half of the proof
/// ([`metadata_labels_keep_series_apart`]) holds, nothing at or below the drop has collected already
/// ([`canonical_below`]), and the only reason a collection is still needed is the scan's key columns,
/// a primary-key column that holds `NULL` in one series and `''` in another. Those two decode to an
/// absent and to an empty label, and the drop normalizes them into one. Such a statement is now
/// collected for that reason alone ([`metadata_drop_keeps_series_apart`]), and this is the count
/// that says so.
///
/// Every node that drops the metadata labels on its own counts: a map of math, a histogram map, a
/// parameterized map and an arithmetic or `bool` scalar broadcast, and the three that rewrite a
/// sample's own value - a name-dropping range function, `timestamp(x)` and a `timestamp` over an
/// evaluated input.
fn drops_metadata_labels_over_one_metric(
node: &NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> bool {
// The statement is counted exactly when the drop is collected for the key columns alone: the
// labels prove it ([`metadata_labels_keep_series_apart`]), nothing at or below it has collected
// already ([`canonical_below`]), and the keys do not survive it - which is the one fact the pass
// read from the resolved table and the reason the merge is there.
let decided_by_the_keys = |carried: &NativeNode| {
!keys_survive
&& metadata_labels_keep_series_apart(carried, source_tags, source_metrics)
&& !canonical_below(carried)
};
let dropped = match &node.expr {
NativeExpr::Instant { input, op } => {
op.value == InstantValue::Timestamp && decided_by_the_keys(input)
}
NativeExpr::Timestamp { input } => decided_by_the_keys(input),
NativeExpr::Window { input, op } => {
op.function.drops_metric_name() && decided_by_the_keys(input)
}
NativeExpr::ParameterizedMap { input, .. } => decided_by_the_keys(input),
NativeExpr::Map { input, op } => {
op.ops
.iter()
.any(|operation| matches!(operation, MapOp::Math(_) | MapOp::Histogram(_)))
&& decided_by_the_keys(input)
}
// A broadcast reaches each series of its vector side as it is; an arithmetic one and one
// that reports a `bool` drop the metadata labels, a filtering comparison keeps them.
NativeExpr::Binary {
left,
right,
matching,
} => {
let carried = match (
left.output.samples().is_some(),
right.output.samples().is_some(),
) {
(true, false) => Some(left.as_ref()),
(false, true) => Some(right.as_ref()),
_ => None,
};
carried.is_some_and(|carried| {
(!matching.op.is_comparison() || matching.return_bool)
&& decided_by_the_keys(carried)
})
}
_ => false,
};
dropped
|| node.expr.children().into_iter().any(|child| {
drops_metadata_labels_over_one_metric(child, source_tags, source_metrics, keys_survive)
})
}
/// Whether `tree` holds a node that drops the metadata labels over series whose labels are proved to
/// tell apart and whose keys do not (see [`drops_metadata_labels_over_one_metric`]).
///
/// A statement this answers `true` for is one the canonical pass collects for the key columns alone:
/// the labels prove the drop injective, nothing below it has collected already, and the scan below it
/// keys its series by a column that can hold `NULL` in one series and `''` in another
/// (`keys_survive` is `false`, and the caller states that fact from the resolved table). Those
/// statements are the price of reading that fact instead of the label-set check the physical scan
/// only states for itself, so the caller counts them by this answer.
pub fn leaves_a_drop_to_the_storage_keys(
tree: &NativeTree,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
keys_survive: bool,
) -> bool {
drops_metadata_labels_over_one_metric(tree.root(), source_tags, source_metrics, keys_survive)
}
/// The one metric every source `node` reads is bound to, or `None` when they are not all one.
///
/// This is the facts-free question about the leaves alone: a node can drop the metric name on the
/// way (a math map, a range function) whatever its sources hold, so what the surviving labels tell
/// apart is only as good as the set of metrics below them.
fn single_source_metric(
node: &NativeNode,
source_metrics: &[Option<Arc<str>>],
) -> Option<Arc<str>> {
let mut pending = vec![node];
let mut metric: Option<&Arc<str>> = None;
while let Some(node) = pending.pop() {
match &node.expr {
// A source that may hand over several metrics proves nothing about their names.
NativeExpr::Input(input) => {
let bound = source_metrics.get(input.source_index)?.as_ref()?;
match metric {
None => metric = Some(bound),
Some(seen) if seen == bound => {}
Some(_) => return None,
}
}
NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => {}
NativeExpr::Instant { input, .. }
| NativeExpr::Window { input, .. }
| NativeExpr::Map { input, .. }
| NativeExpr::Timestamp { input }
| NativeExpr::VectorOf { input }
| NativeExpr::Merge { input }
| NativeExpr::Aggregate { input, .. }
| NativeExpr::Vector { input, .. }
| NativeExpr::ScalarOf { input } => pending.push(input),
NativeExpr::ParameterizedMap {
input, parameters, ..
}
| NativeExpr::ParameterizedHistogram {
input, parameters, ..
} => {
pending.push(input);
pending.extend(parameters.iter());
}
NativeExpr::ParameterizedTopK {
input, parameter, ..
} => {
pending.push(input);
pending.push(parameter);
}
NativeExpr::Binary { left, right, .. }
| NativeExpr::Set { left, right, .. }
| NativeExpr::Info {
base: left,
info: right,
..
} => {
pending.push(left);
pending.push(right);
}
}
}
metric.cloned()
}
/// The one metric every series `node` hands over is named, or `None` when this walk cannot say.
///
/// Only the nodes that hand their input's label set on unchanged can carry a name: every kernel that
/// reports a value of its own drops it, so a name above one of those is one a rewrite wrote. What is
/// left is the boundary the leaves are bound to - where `source_metrics` answers, indexed by the
/// `source_index` the leaf reads - and the nodes that pass a set through.
fn single_metric_name(node: &NativeNode, source_metrics: &[Option<Arc<str>>]) -> Option<Arc<str>> {
match &node.expr {
NativeExpr::Input(input) => source_metrics.get(input.source_index).cloned().flatten(),
// These report the label set they read rather than one of their own, so the name is their
// input's. A map that computes a value or writes a name is not one of them.
NativeExpr::Merge { input }
| NativeExpr::ParameterizedTopK { input, .. }
| NativeExpr::Vector {
input,
op:
BoundVectorOp::TopK { .. }
| BoundVectorOp::LimitK { .. }
| BoundVectorOp::LimitRatio { .. }
| BoundVectorOp::Sort(_),
} => single_metric_name(input, source_metrics),
NativeExpr::Instant { input, op } if op.value != InstantValue::Timestamp => {
single_metric_name(input, source_metrics)
}
NativeExpr::Window { input, op } if !op.function.drops_metric_name() => {
single_metric_name(input, source_metrics)
}
NativeExpr::Map { input, op }
if op.ops.iter().all(|operation| {
matches!(
operation,
MapOp::Label(rewrite) if rewrite.destination() != METRIC_NAME
)
}) =>
{
single_metric_name(input, source_metrics)
}
// A kernel that reports a value of its own drops the name, and a fold, a set operation, an
// `info` lookup and a rewrite writing the name answer a set this walk does not follow.
_ => None,
}
}
/// Whether `node` reads the whole input vector at a point rather than one series at a time.
///
/// An `Aggregate` folds every series of its group, a `Binary` and a `Set` pair two vectors' series
@@ -1871,16 +1464,11 @@ fn single_metric_name(node: &NativeNode, source_metrics: &[Option<Arc<str>>]) ->
/// Both a region-side placement and the native route's admission ask this, and they must agree: a
/// tree admitted here but refused there becomes a node the physical layer cannot build, and one
/// refused here but admitted there would be pushed to a region that holds part of the metric.
///
/// The walk asks the facts-free form of [`unifies_its_own_contributions`]: a placement reads a tree
/// without the binding that says which metric each of its inputs reads, and every reader of this
/// answer - the admission below, the region-side classification and the locality of a built node -
/// asks it of the same tree, so the conservative answer is the one that keeps them agreeing.
pub fn reads_across_series(node: &NativeNode) -> bool {
match &node.expr {
NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => false,
NativeExpr::Window { input, .. } => {
matches!(unifies_its_own_contributions(input, &[], &[]), Some(false))
matches!(unifies_its_own_contributions(input), Some(false))
|| reads_across_series(input)
}
NativeExpr::Instant { input, .. }
@@ -2904,37 +2492,8 @@ fn broadcast_drops_labels(matching: &BinaryMatching) -> bool {
///
/// The collection condition is `matches!(answer, Some(false))`: the Window `MergeInput` collector
/// at runtime and the `limitk` collector both ask this predicate of their input.
///
/// `source_tags` and `source_metrics` are the same facts
/// [`duplicates_canonical_labels`] judges a whole label set by, indexed the same way: a node that
/// drops the metadata labels reports fragments only when
/// [`metadata_labels_keep_series_apart`] cannot prove the labels tell its series apart. That is the
/// labels half of the drop's proof, and it is the half a tree can answer about itself. The other half
/// is whether the scan's key columns can spell one series as `NULL` and another as `''`, both of
/// which the drop normalizes into one label set: it is stated for the tree as a whole and is asked by
/// the canonical pass ([`duplicates_canonical_labels`], which reads it as
/// [`metadata_drop_keeps_series_apart`]). Where that half does not hold, the pass has put its merge
/// above this node already, so this predicate is asked of the merge instead, which is why asking the
/// labels half here cannot disagree with the pass about a drop. The statements that half collects
/// are counted by [`leaves_a_drop_to_the_storage_keys`].
///
/// This predicate is only ever asked of a node by the consumer directly above it - the `Window`
/// `MergeInput` collector and the `limitk` one, both in [`execute`](crate::native::execute) - and
/// that is exactly where the canonical pass puts its [`NativeExpr::Merge`] when a collection is
/// needed: it collects the input of a `Window`, an `Aggregate`, a `Vector`, a `Map`, a
/// `ParameterizedMap` and each side of a `Binary`/`Set`/`Info`
/// ([`merge_label_rewrites`](crate::native::canonicalize::merge_label_rewrites)). The collector
/// then reads the merge's `Some(true)` - a merge *is* the collection - so the weaker answer this
/// function gives about the node beneath it is never the one acted on. Where the pass left the tree
/// alone, the proof held, and the answer asked here is the same one. A caller without these facts
/// (placement, which reads a tree without a binding) passes empty slices and gets the conservative
/// answer.
#[allow(clippy::match_same_arms)] // Keep each expression's contribution guarantee explicit.
pub fn unifies_its_own_contributions(
node: &NativeNode,
source_tags: &[Vec<String>],
source_metrics: &[Option<Arc<str>>],
) -> Option<bool> {
pub fn unifies_its_own_contributions(node: &NativeNode) -> Option<bool> {
match &node.expr {
// A boundary hands over the pieces some upstream execution produced: what it holds is one
// storage series at a time for a scan and one partition's fragments of a label set for an
@@ -2951,20 +2510,7 @@ pub fn unifies_its_own_contributions(
// arrive one whole storage series at a time, so it answers one series per series it read,
// under the label set it read. `timestamp(x)` is the exception: it reports the sample's own
// time under the metric name dropped, which can give two series one label set.
NativeExpr::Instant { input, op } => {
if op.value == InstantValue::Timestamp {
// `timestamp` drops the metadata labels, so what it emits aliases unless the drop
// provably tells its series apart. The labels half is asked here; the keys half is
// what the pass collects with, above this node - see this function's docs.
Some(metadata_labels_keep_series_apart(
input,
source_tags,
source_metrics,
))
} else {
Some(true)
}
}
NativeExpr::Instant { op, .. } => Some(op.value != InstantValue::Timestamp),
// The merge is the collection: whatever its input handed over, it groups the fragments by
// canonical label set and emits one series per set, so what it emits shares none.
NativeExpr::Merge { .. } => Some(true),
@@ -2982,46 +2528,20 @@ pub fn unifies_its_own_contributions(
// folds each contribution on its own, so what it emits does alias its input's fragments,
// and `Some(false)` is exact.
NativeExpr::Window { input, op } => {
let child = unifies_its_own_contributions(input, source_tags, source_metrics);
if child.is_none() {
// A boundary hands the pieces of one series over one at a time, and the window folds
// them without ever seeing the whole series: what it emits aliases whatever the
// labels say.
Some(false)
} else if op.function.drops_metric_name() {
// The function drops the metadata labels, so the window answers `Some(true)` exactly
// when the labels say the drop tells its input's series apart. The keys half is the
// pass's to ask: where it fails, the pass has already put its merge above this
// window and the collector reads that merge's answer - see this function's docs.
Some(metadata_labels_keep_series_apart(
input,
source_tags,
source_metrics,
))
} else {
Some(true)
}
Some(!op.function.drops_metric_name() && unifies_its_own_contributions(input).is_some())
}
// A map can rewrite labels, so two series can end under one label set: what it hands out is
// not one series per set until a canonical merge has collected the set.
// A map can rewrite labels, so two series can end under one label set and the runtime
// records what it wrote in an `Emitted` table.
NativeExpr::Map { .. } | NativeExpr::ParameterizedMap { .. } => Some(false),
// `topk` hands back the series it read with their labels, so the input's answer is its own -
// exactly as for the bound operation.
NativeExpr::ParameterizedTopK { input, .. } => {
unifies_its_own_contributions(input, source_tags, source_metrics)
}
NativeExpr::ParameterizedTopK { input, .. } => unifies_its_own_contributions(input),
// A histogram function groups its rows by label set itself, so it emits one series per set.
NativeExpr::ParameterizedHistogram { .. } => Some(true),
// `scalar()` answers one value per point, which is not several series.
NativeExpr::ScalarOf { .. } => Some(true),
// A timestamp reports a position under the metadata labels dropped, and the labels half
// asked here is answered for the same reason the `timestamp` selection's is: the keys half
// is the one the pass turns into a merge above this node - see this function's docs.
NativeExpr::Timestamp { input } => Some(metadata_labels_keep_series_apart(
input,
source_tags,
source_metrics,
)),
// A timestamp reports a position under the metric name dropped.
NativeExpr::Timestamp { .. } => Some(false),
NativeExpr::Binary {
left,
right,
@@ -3055,7 +2575,7 @@ pub fn unifies_its_own_contributions(
!matches!(node.expr, NativeExpr::Scalar(BoundScalar { .. }))
};
let vector = if carries_series(left) { left } else { right };
unifies_its_own_contributions(vector, source_tags, source_metrics)
unifies_its_own_contributions(vector)
}
}
// A set operation keeps each side's samples and files them by label set, one series per
@@ -3083,7 +2603,7 @@ pub fn unifies_its_own_contributions(
if whole {
Some(true)
} else {
unifies_its_own_contributions(input, source_tags, source_metrics)
unifies_its_own_contributions(input)
}
}
}
+39 -315
View File
@@ -24,8 +24,7 @@ use super::{
AggregateKind, AggregateStage, BoundAggregate, BoundInput, BoundInstant, BoundMap, BoundScalar,
BoundVectorOp, BoundWindow, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree,
NativeType, ParameterizedOp, aggregate_state_binding, duplicates_canonical_labels,
fused_output_tags, leaves_a_drop_to_the_storage_keys, reads_across_series,
unifies_its_own_contributions,
fused_output_tags, reads_across_series, unifies_its_own_contributions,
};
use crate::extension_plan::FieldKind;
use crate::extension_plan::{FusedSeries, FusedSeriesBinding};
@@ -188,44 +187,6 @@ fn window_over(inner: NativeNode, input_grid: Option<Arc<EvalGrid>>) -> NativeNo
window_on(inner, input_grid, grid(0, 60_000, 4))
}
/// An instant selection over `inner` reporting the sample it read, which keeps its identity.
fn instant_sample(inner: NativeNode, out: Arc<EvalGrid>) -> NativeNode {
NativeNode {
expr: NativeExpr::Instant {
input: Box::new(inner),
op: BoundInstant {
lookback_ms: 300_000,
offset_ms: 0,
at_ms: None,
value: InstantValue::Sample,
output_grid: Arc::clone(&out),
},
},
output: evaluated(fields(1), out),
}
}
/// A window over `inner` whose function keeps the metric name, so it hands its input's identity on.
fn keeping_window_fields(inner: NativeNode, out: Arc<EvalGrid>, count: usize) -> NativeNode {
NativeNode {
expr: NativeExpr::Window {
input: Box::new(inner),
op: BoundWindow {
function: RangeFunction::LastOverTime,
window: RangeWindow {
range_ms: 300_000,
offset_ms: 0,
anchor_ms: None,
modifier: None,
},
input_grid: None,
output_grid: Arc::clone(&out),
},
},
output: evaluated(fields(count), out),
}
}
/// A window over `inner` that produces `count` fields on `out`, so a consumer of it can be given a
/// layout the window itself holds legally.
fn window_fields(inner: NativeNode, out: Arc<EvalGrid>, count: usize) -> NativeNode {
@@ -1214,12 +1175,12 @@ fn a_broadcast_side_contributes_no_labels() {
// broadcast reaches each series of its vector side as it is, so whether two series can end
// under one label set is that side's answer, not the constant's.
assert!(
duplicates_canonical_labels(&aliasing(), &source_tags, &[], true),
duplicates_canonical_labels(&aliasing(), &source_tags),
"the fixture aliases: a rewrite of a label its input carries can fold two series into one"
);
assert_eq!(
duplicates_canonical_labels(&aliasing(), &source_tags, &[], true),
duplicates_canonical_labels(&clamp_max(aliasing()), &source_tags, &[], true),
duplicates_canonical_labels(&aliasing(), &source_tags),
duplicates_canonical_labels(&clamp_max(aliasing()), &source_tags),
"a scalar parameter carries no labels, so the map answers its input's identity"
);
for node in [
@@ -1229,8 +1190,8 @@ fn a_broadcast_side_contributes_no_labels() {
binary(eval_time(), clamp_max(aliasing())),
] {
assert_eq!(
duplicates_canonical_labels(&aliasing(), &source_tags, &[], true),
duplicates_canonical_labels(&node, &source_tags, &[], true),
duplicates_canonical_labels(&aliasing(), &source_tags),
duplicates_canonical_labels(&node, &source_tags),
"a broadcast keeps the series identity of its vector side"
);
}
@@ -1243,7 +1204,7 @@ fn a_broadcast_side_contributes_no_labels() {
binary(vector_of_one(), aliasing()),
] {
assert!(
!duplicates_canonical_labels(&node, &source_tags, &[], true),
!duplicates_canonical_labels(&node, &source_tags),
"a vector-vector match answers one series per label set, whatever aliases below it"
);
}
@@ -2200,25 +2161,15 @@ fn scan_tags() -> Vec<Vec<String>> {
vec![vec!["host".to_string()]]
}
/// The labels a scan bound to one metric reports: its tag, and the name its binding gives every
/// series it hands over, which is not one of its tag columns.
fn named_scan_tags() -> Vec<Vec<String>> {
vec![vec!["host".to_string(), "__name__".to_string()]]
}
/// A rewrite that writes a label the input does not carry cannot make two series equal, which is
/// what lets a query like `label_replace(x, "dst", "$1", "host", "(.*)")` keep every partition.
#[test]
fn a_fresh_destination_does_not_alias() {
let out = grid(0, 60_000, 4);
// A window that keeps the metric name hands its input's identity on, so the rewrite is the only
// node that could make two series equal.
let inner = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1);
let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1);
assert!(!duplicates_canonical_labels(
&rewritten(inner, "dst", "host"),
&scan_tags(),
&[],
true
&scan_tags()
));
}
@@ -2230,9 +2181,7 @@ fn a_rewrite_of_an_existing_label_aliases() {
let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1);
assert!(duplicates_canonical_labels(
&rewritten(inner, "host", "host"),
&scan_tags(),
&[],
true
&scan_tags()
));
}
@@ -2246,26 +2195,20 @@ fn a_rewrite_of_the_metric_name_aliases_a_named_source() {
let named = vec![vec!["host".to_string(), "__name__".to_string()]];
assert!(duplicates_canonical_labels(
&rewritten(inner, "__name__", "__name__"),
&named,
&[],
true
&named
));
// The delete-only form can fold the same two identities, so it is collected too.
let inner = input_on(Arc::clone(&out));
assert!(duplicates_canonical_labels(
&replaced(inner, "__name__", "", "__name__"),
&named,
&[],
true
&named
));
}
/// A name write is collected even above a kernel that dropped the name: the scan below `abs`, a
/// `timestamp` or a range function can still hold several metrics whose ordinary labels are equal,
/// so writing one name puts those identities under one label set. A fresh ordinary destination over
/// a kernel that keeps the name is still uncollected, because no scan can carry an ordinary label it
/// does not declare -- while over a name-dropping kernel the answer is `true` for the kernel's own
/// sake, which is what [`dropping_the_metric_name_is_a_new_source`] pins.
/// so writing one name puts those identities under one label set. A fresh ordinary destination is
/// still uncollected, because no scan can carry an ordinary label it does not declare.
#[test]
fn a_name_rewrite_above_a_dropped_name_is_collected() {
let out = grid(0, 60_000, 4);
@@ -2288,21 +2231,16 @@ fn a_name_rewrite_above_a_dropped_name_is_collected() {
] {
assert!(
duplicates_canonical_labels(
&replaced(dropped, "__name__", "same", "missing"),
&named,
&[],
true
&replaced(dropped.clone(), "__name__", "same", "missing"),
&named
),
"{name}: a name write may alias metrics the kernel left indistinguishable"
);
assert!(
!duplicates_canonical_labels(&rewritten(dropped, "dst", "host"), &named),
"{name}: a fresh ordinary destination cannot alias"
);
}
// The fresh destination is judged against a kernel that keeps the name, which is the only shape
// where the rewrite alone decides the answer.
let keeping = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1);
assert!(
!duplicates_canonical_labels(&rewritten(keeping, "dst", "host"), &named, &[], true),
"a fresh ordinary destination cannot alias"
);
}
/// The label walk keeps a name a rewrite writes and drops it wherever the kernel does, so the
@@ -2474,189 +2412,20 @@ fn an_unknown_source_may_alias() {
let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1);
assert!(duplicates_canonical_labels(
&rewritten(inner, "dst", "host"),
&[],
&[],
true
&[]
));
}
/// Dropping the metric name is a source of duplicates in its own right, and a window that keeps it
/// is not.
///
/// The name is part of the identity a merge groups by, so a kernel that computes a value of its own
/// and reports the name dropped can leave two series under one label set - but only when something
/// below it could be told apart by that name.
///
/// A scan bound to one metric names every series it hands over, so no name of its tells two of them
/// apart, and the labels are the whole proof: the one metric is enough - as long as the scan's key
/// columns always decode a value. A drop removes every empty label value with the name, so a key
/// column that holds `NULL` in one series and `''` in another leaves both under one label set and
/// that half withholds the proof too. A scan that may hand over several metrics is collected, and so
/// is one carrying `__type__` or `__unit__` among its tags, because only those labels told the two
/// series apart.
/// Dropping the metric name is not aliasing within one metric: the remaining tags are the primary
/// key, so two series that agree on all of them are the same stored series.
#[test]
fn dropping_the_metric_name_is_a_new_source() {
fn dropping_the_metric_name_keeps_the_inputs_answer() {
let out = grid(0, 60_000, 4);
let single = [Some(Arc::<str>::from("some_metric"))];
let dropped = window_fields(input_with_fields(1), Arc::clone(&out), 1);
// A caller that says nothing about the metric is not proved.
assert!(duplicates_canonical_labels(
&dropped,
&named_scan_tags(),
&[],
true
));
// The scan reads one metric, so the name it drops told none of its series apart.
let plain = window_fields(input_with_fields(1), Arc::clone(&out), 1);
assert!(!duplicates_canonical_labels(&plain, &scan_tags()));
assert!(!duplicates_canonical_labels(
&dropped,
&named_scan_tags(),
&single,
true
));
// A source that may hand over several metrics is not proved, and neither is one that carries a
// metadata label of its own: both can hold two series that only that label told apart.
assert!(duplicates_canonical_labels(
&dropped,
&named_scan_tags(),
&[None],
true
));
for metadata in ["__type__", "__unit__"] {
let mut tags = named_scan_tags();
tags[0].push(metadata.to_string());
assert!(
duplicates_canonical_labels(&dropped, &tags, &single, true),
"{metadata}"
);
}
// The same scan, with a key column that can hold `NULL` in one series and `''` in another: the
// empty value leaves with the name, so the one metric no longer proves the drop - and neither
// does it for `abs` or a `timestamp` above it, which report the name dropped as well.
assert!(duplicates_canonical_labels(
&dropped,
&named_scan_tags(),
&single,
false
));
assert!(duplicates_canonical_labels(
&map_over(input_on(Arc::clone(&out))),
&named_scan_tags(),
&single,
false
));
// A window that keeps the name passes its input's answer on instead.
let keeping = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1);
assert!(!duplicates_canonical_labels(
&keeping,
&named_scan_tags(),
&[],
true
));
// `timestamp` reports a sample's own time under the name dropped, in both of its forms.
let stamps = |inner: NativeNode| NativeNode {
expr: NativeExpr::Timestamp {
input: Box::new(inner),
},
output: evaluated(fields(1), Arc::clone(&out)),
};
assert!(duplicates_canonical_labels(
&stamps(input_on(Arc::clone(&out))),
&named_scan_tags(),
&[],
true
));
assert!(duplicates_canonical_labels(
&stamps(input_on(Arc::clone(&out))),
&named_scan_tags(),
&[None],
true
));
assert!(!duplicates_canonical_labels(
&stamps(input_on(Arc::clone(&out))),
&named_scan_tags(),
&single,
true
));
// A sample-preserving selection keeps the name it read, so it is no source of its own.
assert!(!duplicates_canonical_labels(
&instant_sample(input_on(Arc::clone(&out)), Arc::clone(&out)),
&named_scan_tags(),
&[],
true
));
}
/// The plan-time counter's question: which statements drop the metadata labels over series whose
/// labels prove them apart and whose keys do not.
///
/// The labels half is what the walk reads; the key half is stated by the caller, and a drop is
/// counted exactly when the labels prove it, nothing below it has collected already, and the scan's
/// key columns cannot tell the pair apart - a key column holding `NULL` in one series and `''` in
/// another, which is a fact about the physical table. This is the tree half of that count: the
/// planner pairs it with the fact it read from the resolved table.
#[test]
fn a_drop_the_labels_prove_is_counted_only_when_its_keys_do_not() {
let out = grid(0, 60_000, 4);
let single = [Some(Arc::<str>::from("some_metric"))];
let named = named_scan_tags();
// `rate(x[5m])` and `abs(abs(x))` over one metric: the drop removes a name every series shares,
// and nothing below either node is collected, so the key columns are what decides the answer.
for keys in [false, true] {
let counted = !keys;
assert_eq!(
counted,
leaves_a_drop_to_the_storage_keys(
&NativeTree::try_new(window_fields(input_with_fields(1), Arc::clone(&out), 1))
.unwrap(),
&named,
&single,
keys,
),
"`rate(x[5m])` with keys_survive = {keys}"
);
assert_eq!(
counted,
leaves_a_drop_to_the_storage_keys(
&NativeTree::try_new(map_over(map_over(input_on(Arc::clone(&out))))).unwrap(),
&named,
&single,
keys,
),
"`abs(abs(x))` with keys_survive = {keys}"
);
}
// A source that may hand over several metrics is collected by the labels, so nothing is left to
// the key columns.
assert!(!leaves_a_drop_to_the_storage_keys(
&NativeTree::try_new(window_fields(input_with_fields(1), Arc::clone(&out), 1)).unwrap(),
&named,
&[None],
false,
));
// A window that keeps the name drops nothing.
assert!(!leaves_a_drop_to_the_storage_keys(
&NativeTree::try_new(keeping_window_fields(
input_with_fields(1),
Arc::clone(&out),
1
))
.unwrap(),
&named,
&single,
false,
));
// A merge below the drop hands over one series per canonical label set, which is unique even
// where the grouping treats an empty label value as no label, so the key columns decide nothing.
assert!(!leaves_a_drop_to_the_storage_keys(
&NativeTree::try_new(window_on(
merge_over(input_on(Arc::clone(&out))),
Some(Arc::clone(&out)),
Arc::clone(&out),
))
.unwrap(),
&named,
&single,
false,
&map_over(rewritten(plain, "dst", "host")),
&scan_tags()
));
}
@@ -2665,8 +2434,7 @@ fn a_drop_the_labels_prove_is_counted_only_when_its_keys_do_not() {
#[test]
fn a_rewrite_after_ranking_or_grouping_may_alias() {
let out = grid(0, 60_000, 4);
// A name-keeping window, so the ranking below is the only node whose answer is in question.
let selected = || keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1);
let selected = || window_fields(input_with_fields(1), Arc::clone(&out), 1);
let vector = |op| NativeNode {
expr: NativeExpr::Vector {
input: Box::new(selected()),
@@ -2707,10 +2475,7 @@ fn a_rewrite_after_ranking_or_grouping_may_alias() {
),
("sum by host", grouped_sum),
] {
assert!(
!duplicates_canonical_labels(&inner, &source_tags, &[], true),
"{name}"
);
assert!(!duplicates_canonical_labels(&inner, &source_tags), "{name}");
let output = inner.output.clone();
let renamed = NativeNode {
expr: NativeExpr::Map {
@@ -2726,7 +2491,7 @@ fn a_rewrite_after_ranking_or_grouping_may_alias() {
};
assert!(renamed.validate().is_ok(), "{name}");
assert!(
duplicates_canonical_labels(&renamed, &source_tags, &[], true),
duplicates_canonical_labels(&renamed, &source_tags),
"{name}"
);
}
@@ -2846,9 +2611,7 @@ fn a_merge_answers_one_series_per_set() {
);
assert!(!duplicates_canonical_labels(
&merge_over(aliasing),
&scan_tags(),
&[],
true
&scan_tags()
));
}
@@ -2898,64 +2661,25 @@ fn a_window_unifies_its_own_contributions() {
] {
assert_eq!(
child_answer,
unifies_its_own_contributions(&child, &[], &[]),
unifies_its_own_contributions(&child),
"{name}: the child's own answer"
);
assert_eq!(
window_answer,
unifies_its_own_contributions(&window_of(child, RangeFunction::LastOverTime), &[], &[]),
unifies_its_own_contributions(&window_of(child, RangeFunction::LastOverTime)),
"{name}: a name-keeping window unifies whatever its child reported, and only a \
boundary leaves it aliasing"
);
}
// A function that drops the metadata labels can give two series one label set, so the window
// reports it whatever its child said - and a caller that cannot say which metric the child reads
// has no proof to offer on the child's behalf.
for child in [collected(), fragments()] {
// A function that drops the metric name can give two series one label set whatever its child
// answered, so the window reports it whatever the child said.
for child in [collected(), fragments(), boundary()] {
assert_eq!(
Some(false),
unifies_its_own_contributions(&window_of(child, RangeFunction::Rate), &[], &[]),
"a name-dropping window over an unproved source may alias"
unifies_its_own_contributions(&window_of(child, RangeFunction::Rate)),
"a name-dropping window may alias"
);
}
// A boundary is the one child the proof says nothing about: the window folds the pieces of one
// series one at a time, so what it emits aliases them whatever the labels say.
assert_eq!(
Some(false),
unifies_its_own_contributions(&window_of(boundary(), RangeFunction::Rate), &[], &[]),
"a boundary leaves the window's own answer aliasing"
);
// The proof is the one the canonical pass reads: a child whose labels lost the name, over a
// source bound to a single metric without a metadata tag of its own, keeps the name-dropping
// window from reporting fragments.
let named = [vec!["host".to_string(), "__name__".to_string()]];
let single = [Some(Arc::<str>::from("some_metric"))];
let dropped = map_over(input_on(Arc::clone(&out)));
assert_eq!(
Some(true),
unifies_its_own_contributions(
&window_of(dropped.clone(), RangeFunction::Rate),
&named,
&single
),
"a single metric's dropped name tells none of its series apart"
);
assert_eq!(
Some(false),
unifies_its_own_contributions(
&window_of(dropped.clone(), RangeFunction::Rate),
&named,
&[None]
),
"a source that may hand over several metrics is not proved"
);
let mut typed = named.clone();
typed[0].push("__type__".to_string());
assert_eq!(
Some(false),
unifies_its_own_contributions(&window_of(dropped, RangeFunction::Rate), &typed, &single),
"a metadata tag the window drops is not proved constant"
);
}
/// `info` reads its info side through a canonical merge: the merge only collects the pieces of one
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The grid positions a field occupies, as sorted, disjoint runs.
//!
//! A node that rewrites labels — a `Map` dropping the metric name, a `Window` dropping the name and
//! the empty-valued labels — can turn two input series into one label set, and every layer above
//! then reads the two chunks as one series. What decides whether that pair is legal is which
//! positions of each field already hold a value: two values of one field at one position cannot be
//! merged away, while a chunk that fills positions the set left empty merges cleanly, as
//! [`crate::grid_series::merge_same_labelset`] does.
//!
//! The native runtime's partition-local [`crate::native::execute::Emitted`] table keeps this state
//! per label set and field as runs, so a chunk of many positions costs one run rather than one
//! entry per position.
/// One run of grid positions, half-open, that hold a value in one field.
#[derive(Clone, Copy, Debug)]
pub(crate) struct Run {
pub(crate) start: usize,
pub(crate) end: usize,
}
impl Run {
/// Whether two runs share a position.
pub(crate) fn overlaps(self, other: Self) -> bool {
self.start < other.end && other.start < self.end
}
}
// How many times `absorb` rebuilt the stored runs instead of appending to them.
//
// Test-only, and per thread: the runs stored afterwards are the same either way, so a rebuild shows
// only in what it costs, and a test that reads its own thread's figure is not disturbed by another
// test's chunks.
#[cfg(test)]
thread_local! {
static ABSORB_REBUILDS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// How many times this thread's [`absorb`] rebuilt the stored runs.
#[cfg(test)]
pub(crate) fn absorb_rebuilds() -> usize {
ABSORB_REBUILDS.with(std::cell::Cell::get)
}
/// Merges `incoming` into `stored`, keeping the result in position order and joining the runs that
/// touch.
///
/// Within a chunk one pass over both sides is what keeps this linear, since each side is sorted and
/// disjoint. Adding the runs one at a time and rebuilding the array after each would cost the square
/// of the runs a chunk yields: a chunk of one hundred thousand positions alternating between values
/// and nulls yields fifty thousand runs, and re-merging after each is over a billion element moves
/// for a single series.
///
/// One pass per chunk is still a pass over everything the series has stored so far, so it is not
/// free once the *history* grows across chunks: a series split into `K` chunks of one position each,
/// a hole apart, stores one more run per chunk and moves `K(K+1)/2` of them, with no label collision
/// and no illegal input anywhere. A chunk that starts no earlier than the end of the last stored run
/// only ever adds to the end — the stored runs are sorted and disjoint, so every one of them ends
/// before anything this chunk holds — and it is appended instead, which is amortised constant per
/// run. That is the shape a scan hands over as its positions move forward chunk after chunk. A chunk
/// that starts before the last stored run can split a run or reach back past it, so it still costs
/// the full merge.
///
/// Either way the result is sorted, disjoint, and has the runs that touch joined, which is what a
/// look-up by [`Run::overlaps`] on the runs stored here relies on.
pub(crate) fn absorb(stored: &mut Vec<Run>, incoming: &[Run]) {
if incoming.is_empty() {
return;
}
if stored
.last()
.is_none_or(|last| incoming[0].start >= last.end)
{
append_runs(stored, incoming);
} else {
merge_runs(stored, incoming);
}
}
/// Appends `incoming` to `stored`, all of which lies past the last stored run.
///
/// Nothing already stored has to be placed again: the two sides are each in position order and
/// nothing `incoming` holds can reach back into what is stored, so its runs are pushed on the end and
/// the vector's own growth is the only move — amortised over the runs appended. Where its first run
/// touches the end of the last stored run the two are one run, as they are after a merge, so it is
/// joined rather than left beside it.
fn append_runs(stored: &mut Vec<Run>, incoming: &[Run]) {
let mut incoming = incoming;
if let Some(last) = stored.last_mut()
&& let Some(first) = incoming.first()
&& first.start <= last.end
{
last.end = last.end.max(first.end);
incoming = &incoming[1..];
}
stored.extend_from_slice(incoming);
}
/// Merges `incoming` into `stored`, where the two can interleave.
///
/// One pass is enough because each side is sorted and disjoint: whichever run starts first is
/// placed, and it joins the run before it where the two touch.
fn merge_runs(stored: &mut Vec<Run>, incoming: &[Run]) {
#[cfg(test)]
ABSORB_REBUILDS.with(|rebuilds| rebuilds.set(rebuilds.get() + 1));
let mut merged: Vec<Run> = Vec::with_capacity(stored.len() + incoming.len());
let (mut a, mut b) = (0, 0);
while a < stored.len() || b < incoming.len() {
let larger = match (stored.get(a), incoming.get(b)) {
(Some(left), Some(right)) if right.start < left.start => {
b += 1;
*right
}
(Some(left), _) => {
a += 1;
*left
}
(None, Some(right)) => {
b += 1;
*right
}
(None, None) => break,
};
match merged.last_mut() {
Some(last) if larger.start <= last.end => last.end = last.end.max(larger.end),
_ => merged.push(larger),
}
}
*stored = merged;
}
+15
View File
@@ -268,6 +268,21 @@ fn cut_as_segment(node: &NativeNode, canonical_segmentation: bool) -> bool {
!sources.is_empty()
}
/// Whether any canonical merge in `node` answers with histogram samples, given the kind its input
/// carries.
///
/// A merge collects the pieces of one label set and hands on the samples it was given, so its answer
/// kind is its child's. A histogram answer standing on a merge would travel through the canonical
/// boundary carrying histogram frames, a pairing no execution exercises yet: the planner keeps such
/// a statement on the existing plan until that pairing is covered, and it has to look at every
/// merge, not only a root one.
pub fn merge_answers_histogram(node: &NativeNode, input_kind: FieldKind) -> bool {
node.any(&|node| {
matches!(node.expr, NativeExpr::Merge { .. })
&& node.output_field_kind(Some(input_kind)) == Some(FieldKind::Histogram)
})
}
/// Whether the merge producer's entire prefix can run on each scan partition independently.
pub fn canonical_local_prefix(node: &NativeNode) -> bool {
match &node.expr {
+33 -144
View File
@@ -200,36 +200,12 @@ pub struct RangeEvaluator {
batch: Option<BatchKernel>,
/// Windows accumulated for the next batch call.
pending: PendingBatch,
/// Buffers the bulk ingest shifts and compacts a chunk into, kept between chunks so a series
/// that needs them does not allocate per push.
bulk_scratch: BulkScratch,
/// Batch kernel calls made so far. Test-only: both dispatch shapes produce the same output,
/// so a test comparing only output could not tell which one ran.
#[cfg(test)]
batch_calls: usize,
}
/// The buffers the bulk ingest needs for a chunk whose samples are not retained as they arrive:
/// the timestamps shifted by the selector offset, and one compacted column per field.
#[derive(Default)]
struct BulkScratch {
timestamps: Vec<i64>,
columns: Vec<Vec<f64>>,
}
impl BulkScratch {
/// The bytes these buffers hold, by capacity.
fn bytes(&self) -> usize {
self.timestamps.capacity() * std::mem::size_of::<i64>()
+ self.columns.capacity() * std::mem::size_of::<Vec<f64>>()
+ self
.columns
.iter()
.map(|column| column.capacity() * std::mem::size_of::<f64>())
.sum::<usize>()
}
}
/// Windows accumulated for the next batch call: one list per field.
struct PendingBatch {
/// Grid index of each accumulated point, ascending.
@@ -710,20 +686,11 @@ impl RangeEvaluator {
bulk_ingests: 0,
batch: None,
pending: PendingBatch::with_fields(input.fields.len()),
bulk_scratch: BulkScratch::default(),
#[cfg(test)]
batch_calls: 0,
})
}
/// The evaluation grid this evaluator decides points of.
///
/// Every window ends at a point of it and is read at that point's timestamp, so a caller
/// reserving for what the evaluator is about to allocate reserves this grid's timestamps.
pub fn grid(&self) -> &Arc<EvalGrid> {
&self.grid
}
/// Chunks appended without the per-sample loop. Test-only: both paths produce the same
/// output by construction, so a test that only compared output could not tell whether the
/// bulk path ran at all.
@@ -749,9 +716,8 @@ impl RangeEvaluator {
/// Counted by capacity, so it is what the storage holds rather than what it uses: the retained
/// samples per field (timestamps, values, the reset index and the native histograms with the
/// payload each of them owns), the row a push fills with the histograms it boxes, the identity
/// of the open series, the pending batch's own vectors and the buffers the bulk ingest shifts
/// and compacts into. Left out are the kernel's scratch buffers, this struct's `Vec` headers
/// and the grid, which every chunk addressed on it shares.
/// of the open series and the pending batch's own vectors. Left out are the kernel's scratch
/// buffers, this struct's `Vec` headers and the grid, which every chunk addressed on it shares.
///
/// The figure is an estimate, not a ledger of every allocation: a histogram's payload is
/// measured once when it is admitted and released with it, so neither the retained samples nor
@@ -769,7 +735,6 @@ impl RangeEvaluator {
+ boxed_bytes(&self.row)
+ self.series.as_ref().map_or(0, SeriesState::identity_bytes)
+ self.pending.bytes()
+ self.bulk_scratch.bytes()
}
/// An upper bound of the rows the next push hands out, for input whose last sample is at
@@ -1101,139 +1066,63 @@ impl RangeEvaluator {
Ok(())
}
/// Appends the chunk in bulk when every field is an entirely present float column.
/// Appends the chunk in bulk when every field is an entirely present, unshifted float column.
///
/// Returns `false` when a condition of the per-sample path applies, so the caller can run it.
/// Only the shapes that path alone can read do: a field holding native histogram samples or
/// mixed floats and histograms -- `dense()` reports [`DenseSamplesRef::Mixed`] for the latter,
/// keeping its windows on the histogram path where the mixed-window rules decide the result --
/// and a column with missing samples, which are absent rows rather than values.
/// Returns `false` when any condition of the per-sample path applies, so the caller can run
/// it. A chunk carrying a stale marker takes that path too: the marker drops the whole row,
/// and reproducing that here would cost the same scan the loop already makes.
///
/// A selector offset and a stale marker do not: the shift and the rows the marker drops are
/// applied here, over the columns, instead of handing the chunk to the per-sample loop. The
/// search for a marker is one short-circuiting pass, the same pass the loop would have made.
/// A field mixing floats and native histograms never qualifies: `dense()` reports
/// [`DenseSamplesRef::Mixed`] for it, keeping its windows on the histogram path where the
/// mixed-window rules decide the result.
fn ingest_in_bulk(
&mut self,
timestamps: &[i64],
fields: &SeriesFields,
drop_stale: bool,
) -> Result<bool> {
) -> bool {
// Lengths are checked by `check_chunk` for raw input and by `push_series` for evaluated
// input, so only the shape of the columns is decided here.
if self.window.offset_ms != 0 {
return false;
}
let mut columns = Vec::with_capacity(fields.len());
for field in fields.iter() {
let Some(DenseSamplesRef::Float(values)) = field.dense() else {
return Ok(false);
return false;
};
if values.null_count() != 0 {
return Ok(false);
return false;
}
columns.push(values.values().as_ref());
columns.push(values);
}
// A stale marker drops its whole row, so which rows go is decided before anything is
// retained. A chunk holding none -- the common case -- is judged by one scan that stops at
// the first marker and then leaves the columns as they are.
let stale_rows = if drop_stale
&& columns
.iter()
.any(|values| values.iter().any(|value| is_prometheus_stale_nan(*value)))
// A kept sample's shifted time must fit `i64`; with no offset it is the source timestamp,
// which `RawTimestamps` already validated as increasing and which the frame widened into
// this array without narrowing, so there is nothing left to check here.
if drop_stale
&& columns.iter().any(|values| {
values
.values()
.iter()
.any(|value| is_prometheus_stale_nan(*value))
})
{
let mut stale = vec![false; timestamps.len()];
for values in &columns {
for (row, dropped) in stale.iter_mut().enumerate() {
*dropped |= is_prometheus_stale_nan(values[row]);
}
}
Some(stale)
} else {
None
};
self.append_float_chunk(timestamps, &columns, stale_rows.as_deref())?;
return false;
}
for (field, values) in self.fields.iter_mut().zip(columns) {
field.push_bulk(timestamps, values.values());
}
#[cfg(test)]
{
self.bulk_ingests += 1;
}
Ok(true)
}
/// Retains one chunk of entirely present float columns.
///
/// `stale` marks the rows a stale marker drops, `None` when the chunk holds none. The retained
/// samples are the chunk's rows less those, on the evaluation timeline: the selector offset
/// shifts them, and only a chunk that needs neither keeps the incoming timestamps as they are.
fn append_float_chunk(
&mut self,
timestamps: &[i64],
columns: &[&[f64]],
stale: Option<&[bool]>,
) -> Result<()> {
let offset_ms = self.window.offset_ms;
if offset_ms == 0 && stale.is_none() {
// The source timestamps, which `RawTimestamps` already validated as increasing and
// which the frame widened into this array without narrowing: nothing to shift, nothing
// to check, and the two appends become two `memcpy`s.
for (field, values) in self.fields.iter_mut().zip(columns) {
field.push_bulk(timestamps, values);
}
return Ok(());
}
let BulkScratch {
timestamps: mut shifted,
columns: mut compacted,
} = std::mem::take(&mut self.bulk_scratch);
shifted.clear();
shifted.reserve(timestamps.len());
for (row, &timestamp) in timestamps.iter().enumerate() {
if stale.is_some_and(|stale| stale[row]) {
continue;
}
// Like RangeManipulate, any kept sample whose shifted time leaves i64 fails the series,
// whichever path retains it.
let Some(shifted_timestamp) = timestamp.checked_add(offset_ms) else {
self.bulk_scratch = BulkScratch {
timestamps: shifted,
columns: compacted,
};
return Err(DataFusionError::Execution(format!(
"sample timestamp {timestamp} overflows with offset {offset_ms} ms"
)));
};
shifted.push(shifted_timestamp);
}
match stale {
None => {
for (field, values) in self.fields.iter_mut().zip(columns) {
field.push_bulk(&shifted, values);
}
}
Some(stale) => {
compacted.resize_with(columns.len(), Vec::new);
for (retained, values) in compacted.iter_mut().zip(columns) {
retained.clear();
retained.extend(
values
.iter()
.enumerate()
.filter(|(row, _)| !stale[*row])
.map(|(_, &value)| value),
);
}
for (field, values) in self.fields.iter_mut().zip(&compacted) {
field.push_bulk(&shifted, values);
}
}
}
self.bulk_scratch = BulkScratch {
timestamps: shifted,
columns: compacted,
};
Ok(())
true
}
/// Retains the samples of `fields` at `timestamps`. From `raw` input a row holding a stale
/// marker in any field is dropped; evaluated input keeps it as a value.
fn ingest(&mut self, timestamps: &[i64], fields: &SeriesFields, raw: bool) -> Result<()> {
if self.ingest_in_bulk(timestamps, fields, raw)? {
if self.ingest_in_bulk(timestamps, fields, raw) {
return Ok(());
}
let mut samples: Vec<_> = fields.iter().map(|field| field.iter()).collect();
+12 -219
View File
@@ -18,28 +18,10 @@
//! operation order. Arithmetic that GreptimeDB's range UDFs compute the same way is shared with
//! them.
use std::collections::VecDeque;
use std::ops::Range;
use crate::functions::{EdgeKind, count_edges, extremum, instant_delta, kahan_inc};
use crate::go_sort;
use crate::range_eval::{BatchKernel, FloatWindow, FloatWindows, RangeModifier, WindowKernel};
/// The samples the batch kernels examine, counted per value.
///
/// A batch kernel exists so a field is walked once per batch instead of once per window, which the
/// results cannot show: both shapes compute the same values. Counting what a batch kernel touches
/// is what tells them apart.
#[cfg(test)]
pub(super) static SAMPLES_EXAMINED: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
/// Counts `count` samples examined by a batch kernel; a no-op outside tests.
fn examined(_count: usize) {
#[cfg(test)]
SAMPLES_EXAMINED.fetch_add(_count as u64, std::sync::atomic::Ordering::Relaxed);
}
/// A PromQL range-vector function over float and native histogram samples.
#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)]
pub enum RangeFunction {
@@ -109,34 +91,28 @@ impl RangeFunction {
/// Whether the evaluator has to index counter resets for this function.
///
/// This must hold for exactly the functions that read the index: `rate` and `increase`, whose
/// kernels read [`FloatWindow::counter_resets`] -- the same as saying `IS_COUNTER` in
/// `extrapolated_rate` -- and `resets`, whose batch kernel reads [`FloatWindows::resets_of`].
/// A kernel that corrects or counts counters without the index being maintained would silently
/// miss every reset, so a new reset-reading function has to be added here as well.
/// This must hold for exactly the kernels that read [`FloatWindow::counter_resets`], which is
/// the same as saying `IS_COUNTER` in `extrapolated_rate`: `rate` and `increase`. A kernel
/// that corrects counters without the index being maintained would silently drop every
/// correction, so a new counter function has to be added here as well.
///
/// It governs both readers of the index. When it returns `false`, `resets` stays empty,
/// `resets_of` yields nothing and `counter_resets` is empty, so a kernel reading either would
/// see a series that never reset.
/// It governs both readers of the index: the per-window kernel above, and the batch kernel,
/// which reads [`FloatWindows::resets_of`]. When it returns `false`, `resets` stays empty and
/// `resets_of` yields nothing, so a batch kernel for a counter function would skip every
/// correction and report `delta`-like values.
pub(crate) fn indexes_counter_resets(self) -> bool {
matches!(self, Self::Rate | Self::Increase | Self::Resets)
matches!(self, Self::Rate | Self::Increase)
}
/// The batch form of this function, when it has one.
///
/// A batch form exists for the functions whose window answer is a running summary of the field
/// -- the two endpoints and the resets for the counter rates, a movement of the window's bounds
/// for the extremes, the resets inside it, and the change edges inside it. The rest keep the
/// per-window form, which the evaluator falls back to, so adding a function here is what
/// changes which shape its evaluation takes.
/// Only the counter functions have one. The rest keep the per-window form, which the
/// evaluator falls back to, so adding a function here is what changes which shape its
/// evaluation takes.
pub(crate) fn batch_kernel(self) -> Option<BatchKernel> {
match self {
Self::Rate => Some(Box::new(extrapolated_rate_batch::<true, true>)),
Self::Increase => Some(Box::new(extrapolated_rate_batch::<true, false>)),
Self::MinOverTime => Some(extremum_batch::<true>()),
Self::MaxOverTime => Some(extremum_batch::<false>()),
Self::Resets => Some(Box::new(resets_batch)),
Self::Changes => Some(changes_batch()),
_ => None,
}
}
@@ -512,189 +488,6 @@ pub(super) struct ExtrapolationSpan {
pub samples_minus_one: usize,
}
/// The batch form of `min_over_time` (`IS_MIN`) and `max_over_time`: one pass over the union of
/// the batch's windows, carrying the candidates that can still become the window's extremum.
///
/// The windows of a batch start and end no earlier than the ones before them, so a sample that
/// lost to a later, strictly better one can never win a later window either: it leaves the back of
/// the queue, and a sample the window start has passed leaves the front. That is exactly what
/// [`extremum`] computes window by window, which is why the deque keeps the arrival order of equal
/// values (and with it the sign of tied zeros) and why a NaN never wins while the window holds
/// anything else.
fn extremum_batch<const IS_MIN: bool>() -> BatchKernel {
let mut candidates = SlidingExtremum::<IS_MIN>::default();
Box::new(
move |batch: &FloatWindows<'_>, out: &mut Vec<Option<f64>>| {
out.clear();
out.reserve(batch.windows.len());
// Batches are independent: the retained arrays of the previous one may have been drained
// since, and the samples behind the first window are gone anyway.
candidates.clear();
let values = batch.values;
let sliding = pays_to_slide(batch.windows);
for span in batch.windows {
let (start, end) = (span.start, span.end);
out.push(if sliding {
candidates.evaluate(values, start, end)
} else if start == end {
None
} else {
examined(end - start);
extremum(values[start..end].iter().copied(), IS_MIN)
});
}
},
)
}
/// The samples between the first window's start and the last window's end: what a window-moving
/// summary of a batch has to walk, against the sum of the windows' lengths a rescan walks.
fn union_len(windows: &[Range<usize>]) -> usize {
match (windows.first(), windows.last()) {
(Some(first), Some(last)) => last.end.saturating_sub(first.start),
_ => 0,
}
}
/// Whether walking the union of a batch's windows once beats scanning each window on its own.
///
/// Reuse pays off in proportion to how much consecutive windows overlap: a window-moving summary
/// visits every sample between the first window's start and the last one's end -- plus a step per
/// window -- while a scan visits every sample of every window. On wide steps that barely overlap
/// the walk is the more expensive of the two.
fn pays_to_slide(windows: &[Range<usize>]) -> bool {
let scanned: usize = windows.iter().map(|span| span.len()).sum();
union_len(windows) + windows.len() < scanned
}
/// The extrema candidates of one window, carried to the next.
///
/// Consecutive windows ascend, so the queue is only trimmed at the front and appended at the back.
#[derive(Default)]
struct SlidingExtremum<const IS_MIN: bool> {
/// Indices of the samples that can still become the extremum, in arrival order. A strictly
/// better sample evicts the ones queued before it, so the front is the window's extremum.
candidates: VecDeque<usize>,
/// Backs the all-NaN window, which reports the last NaN it holds, as [`extremum`] does.
latest_nan: Option<usize>,
/// The window the queue describes; `None` until the first window of a batch.
previous: Option<(usize, usize)>,
}
impl<const IS_MIN: bool> SlidingExtremum<IS_MIN> {
fn clear(&mut self) {
self.candidates.clear();
self.latest_nan = None;
self.previous = None;
}
/// Whether `value` replaces `current`, the comparison [`extremum`] replaces on.
#[inline]
fn beats(value: f64, current: f64) -> bool {
if IS_MIN {
value < current
} else {
value > current
}
}
/// The extremum of `values[start..end]`, `None` when the window holds no sample.
fn evaluate(&mut self, values: &[f64], start: usize, end: usize) -> Option<f64> {
let append_start = match self.previous {
Some((previous_start, previous_end))
if start >= previous_start && end >= previous_end =>
{
while self.candidates.front().is_some_and(|&index| index < start) {
self.candidates.pop_front();
}
if self.latest_nan.is_some_and(|index| index < start) {
self.latest_nan = None;
}
previous_end.max(start)
}
// A retreating window would need samples the queue no longer holds. A batch's windows
// ascend, so this is the state a call starts from, not a case the walk reaches.
Some(_) | None => {
self.clear();
start
}
};
for (offset, &value) in values[append_start..end].iter().enumerate() {
if value.is_nan() {
self.latest_nan = Some(append_start + offset);
continue;
}
while let Some(&back) = self.candidates.back() {
if !Self::beats(value, values[back]) {
break;
}
self.candidates.pop_back();
}
self.candidates.push_back(append_start + offset);
}
examined(end.saturating_sub(append_start));
self.previous = Some((start, end));
self.candidates
.front()
.or(self.latest_nan.as_ref())
.map(|&index| values[index])
}
}
/// The batch form of `changes`: a prefix count of the change edges between neighbouring samples,
/// so a window's answer is the difference of two entries instead of a scan.
///
/// The edges are counted with [`count_edges`], the predicate the per-window kernel uses, so the
/// two shapes agree on what a change is -- including that two NaNs, whose comparison is always
/// false, are not one.
fn changes_batch() -> BatchKernel {
let mut prefix: Vec<u64> = Vec::new();
Box::new(
move |batch: &FloatWindows<'_>, out: &mut Vec<Option<f64>>| {
out.clear();
out.reserve(batch.windows.len());
let Some((first, last)) = batch.windows.first().zip(batch.windows.last()) else {
return;
};
let (union_start, union_end) = (first.start, last.end);
let scan = !pays_to_slide(batch.windows);
// `prefix[i]` counts the change edges among `values[union_start..=i]`, so the window
// `[start, end)` counts the edges inside it: its first one is the edge into `start + 1`.
prefix.clear();
if !scan {
prefix.push(0);
examined(union_end.saturating_sub(union_start));
for index in union_start + 1..union_end {
let edge = count_edges(&batch.values[index - 1..=index], EdgeKind::Changes);
prefix.push(prefix[prefix.len() - 1] + edge);
}
}
for span in batch.windows {
out.push((span.start != span.end).then(|| {
if scan {
examined(span.len());
return count_edges(&batch.values[span.start..span.end], EdgeKind::Changes)
as f64;
}
let below = prefix[span.start - union_start];
let inside = prefix[span.end - 1 - union_start];
(inside - below) as f64
}));
}
},
)
}
/// The batch form of `resets`: the resets inside each window, read off the evaluator's reset
/// index. No sample is scanned: a window's answer is a count of index entries between its bounds.
fn resets_batch(batch: &FloatWindows<'_>, out: &mut Vec<Option<f64>>) {
out.clear();
out.reserve(batch.windows.len());
for (index, span) in batch.windows.iter().enumerate() {
out.push((span.start != span.end).then(|| batch.resets_of(index).len() as f64));
}
}
/// The factor `extrapolatedRate` scales the sampled increase by. `zero_point` is the first value
/// over the increase, given only when Prometheus extrapolates a counter to zero: it is the share of
/// the sampled interval that lies before `span.sampled_from`, so `0` says the counter's zero point
+2 -3
View File
@@ -159,8 +159,7 @@ pub(super) fn evaluate(
HistogramAggregate::Avg
};
// The hints are read from the window's own histograms, so nothing is cloned for the
// check; the fold below reads them by reference too and copies only the accumulator
// it starts from.
// check; the fold below clones one sample at a time as it consumes them.
if has_contradictory_reset_hints(histograms.iter().map(|(_, h)| h)) {
record_warning(
collector,
@@ -174,7 +173,7 @@ pub(super) fn evaluate(
// flag is the reconciliation its own adds already observed. Re-walking the pairs to
// learn either would double the histogram work of every window.
let (result, custom_reconciled) =
sum_or_avg(op, histograms.iter().map(|(_, h)| Ok(Cow::Borrowed(h))))?;
sum_or_avg(op, histograms.iter().map(|(_, h)| Ok(h.clone())))?;
if result.is_none() {
record_warning(
collector,
-232
View File
@@ -38,7 +38,6 @@ use datatypes::series::{Label, SeriesLabels};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use super::function::SAMPLES_EXAMINED;
use super::{
CounterReset, FieldSamples, FloatWindow, POSITION_COMPARISONS, PendingBatch, RangeEvaluator,
RangeFunction, RangeModifier, RangeWindow, WindowKernel, accumulate, subquery_grid,
@@ -1924,237 +1923,6 @@ fn a_function_without_a_batch_form_stays_on_the_per_window_path() {
assert_eq!(evaluator.batch_call_count(), 0);
}
/// Rows of a counter that resets twice, plateaus once, steps unevenly and holds one stale marker:
/// every shape an extremum, a reset count or a change count can get wrong at a window bound.
fn summary_rows(points: usize) -> Vec<Row> {
let mut value = 0.0;
(0..points)
.map(|index| {
value = match index {
200 | 500 => 1.0,
300 => value,
_ if index % 3 == 0 => value + 2.0,
_ => value + 1.0,
};
let sample = if index == 150 { stale() } else { value };
(index as i64 * 10_000, vec![Some(sample)])
})
.collect()
}
/// The functions whose window answer is a running summary of the field -- the extremes, the resets
/// and the changes -- reach the batch kernel, and its answers equal the per-window ones.
///
/// The series is longer than `EVALUATION_BATCH`, so the batch also flushes in the middle of the
/// evaluation and has to carry the summary across the flush. A summary read off the wrong window
/// bounds, or a reset counted across one, differs from the per-window result on these rows.
#[test]
fn window_summaries_reach_the_batch_kernel() {
let points = 700;
let rows = summary_rows(points);
let window = RangeWindow {
range_ms: 40_000,
offset_ms: 0,
anchor_ms: None,
modifier: None,
};
for function in [
RangeFunction::MinOverTime,
RangeFunction::MaxOverTime,
RangeFunction::Resets,
RangeFunction::Changes,
] {
let mut batch = RangeEvaluator::try_new(
&raw_schema(1),
grid(0, 10_000, points as u64),
window,
function,
)
.unwrap();
// The per-window path is the same evaluator without the batch kernel, which leaves the
// rest of the evaluation identical.
let mut per_window = RangeEvaluator::try_new(
&raw_schema(1),
grid(0, 10_000, points as u64),
window,
function,
)
.unwrap();
per_window.batch = None;
let piece = chunk(&rows, 1, 0, true);
let batched = collect_chunks(&mut batch, std::slice::from_ref(&piece));
let expected = collect_chunks(&mut per_window, std::slice::from_ref(&piece));
assert!(
batch.batch_call_count() > 0,
"{function:?} never reached its batch kernel"
);
assert_same_series(
fold_chunks(&batched, batch.output_schema()).as_ref(),
fold_chunks(&expected, per_window.output_schema()).as_ref(),
&format!("{function:?}"),
);
}
}
/// A batch kernel walks a field once per batch, not once per grid point.
///
/// The per-window form visits every sample of every window, so `points` windows of `width` samples
/// visit `points * width`. A batch visits the union of its windows once plus one entry per window,
/// so the total stays close to the samples the series holds whatever the width is. The figure is
/// counted rather than timed, which is what makes it a property of the code and not of the
/// machine.
#[test]
fn window_summaries_are_not_rescanned_per_step() {
let points = 2_048u64;
let width = 60;
let rows = summary_rows(points as usize);
let window = RangeWindow {
range_ms: width as i64 * 10_000,
offset_ms: 0,
anchor_ms: None,
modifier: None,
};
// What rescanning each window would visit, and what a walk of the retained samples costs.
let rescanned = points * width;
let once_over = points * 2;
let examined = |function| -> u64 {
SAMPLES_EXAMINED.store(0, std::sync::atomic::Ordering::Relaxed);
let mut evaluator =
RangeEvaluator::try_new(&raw_schema(1), grid(0, 10_000, points), window, function)
.unwrap();
evaluator.push(&chunk(&rows, 1, 0, true)).unwrap();
SAMPLES_EXAMINED.load(std::sync::atomic::Ordering::Relaxed)
};
for function in [
RangeFunction::MinOverTime,
RangeFunction::MaxOverTime,
RangeFunction::Changes,
] {
let visits = examined(function);
assert!(
visits > 0,
"{function:?} reached no batch kernel, so nothing was counted"
);
assert!(
visits <= once_over,
"{function:?} examined {visits} samples over {points} grid points, more than the \
{once_over} a single walk of the series costs: the windows are being rescanned"
);
}
// `resets` reads the reset index, so it touches no sample at all.
assert_eq!(
examined(RangeFunction::Resets),
0,
"resets scanned samples instead of reading the reset index"
);
assert!(rescanned > once_over * 10, "the comparison proves nothing");
}
/// The samples of a chunk kept in segments, which `dense()` refuses: the per-sample loop is the
/// only path that retains them, so one chunk can be handed to either path unchanged.
fn segmented(piece: &RawSeriesChunk, split: usize) -> RawSeriesChunk {
let mut piece = piece.clone();
let len = piece.time.len();
let fields = piece
.fields
.iter()
.map(|field| {
Samples::try_from_chunks(vec![
field.slice(0, split).unwrap(),
field.slice(split, len - split).unwrap(),
])
.unwrap()
})
.collect();
piece.fields = SeriesFields::try_from_vec(fields).unwrap();
piece
}
/// A chunk read with a selector offset is appended in bulk, with the shift applied there rather
/// than by handing the whole chunk to the per-sample loop.
///
/// The same rows go to two evaluators, one from a single dense chunk -- which the bulk path takes
/// -- and one from a chunk kept in segments, which only the per-sample loop retains. The results
/// have to be identical, and only the first may count a bulk ingest.
#[test]
fn an_offset_chunk_is_appended_in_bulk() {
let rows: Vec<Row> = (0..5)
.map(|index| (index as i64 * 10_000, vec![Some(index as f64)]))
.collect();
let window = RangeWindow {
range_ms: 20_000,
offset_ms: 10_000,
anchor_ms: None,
modifier: None,
};
let evaluate = |piece: &RawSeriesChunk| {
let mut evaluator = RangeEvaluator::try_new(
&raw_schema(1),
grid(0, 10_000, 5),
window,
RangeFunction::SumOverTime,
)
.unwrap();
let chunks = collect_chunks(&mut evaluator, std::slice::from_ref(piece));
(
fold_chunks(&chunks, evaluator.output_schema()),
evaluator.bulk_ingest_count(),
)
};
let piece = chunk(&rows, 1, 0, true);
let (bulk, bulk_ingests) = evaluate(&piece);
let (per_sample, per_sample_ingests) = evaluate(&segmented(&piece, 2));
assert_eq!(
bulk_ingests, 1,
"a chunk shifted by an offset was handed to the per-sample loop"
);
assert_eq!(
per_sample_ingests, 0,
"the segmented chunk took the bulk path, so the two were not compared"
);
assert_same_series(bulk.as_ref(), per_sample.as_ref(), "offset");
}
/// A chunk holding a stale marker is appended in bulk too, with the rows the marker drops taken
/// out there, and the result is what retaining the same rows sample by sample produces.
#[test]
fn a_stale_marker_does_not_force_the_per_sample_loop() {
let mut rows: Vec<Row> = (0..5)
.map(|index| (index as i64 * 10_000, vec![Some(index as f64)]))
.collect();
rows[2].1[0] = Some(stale());
let window = RangeWindow {
range_ms: 20_000,
offset_ms: 0,
anchor_ms: None,
modifier: None,
};
let evaluate = |piece: &RawSeriesChunk| {
let mut evaluator = RangeEvaluator::try_new(
&raw_schema(1),
grid(0, 10_000, 5),
window,
RangeFunction::LastOverTime,
)
.unwrap();
let chunks = collect_chunks(&mut evaluator, std::slice::from_ref(piece));
(
fold_chunks(&chunks, evaluator.output_schema()),
evaluator.bulk_ingest_count(),
)
};
let piece = chunk(&rows, 1, 0, true);
let (bulk, bulk_ingests) = evaluate(&piece);
let (per_sample, per_sample_ingests) = evaluate(&segmented(&piece, 2));
assert_eq!(
bulk_ingests, 1,
"a chunk holding a stale marker was handed to the per-sample loop"
);
assert_eq!(per_sample_ingests, 0);
assert_same_series(bulk.as_ref(), per_sample.as_ref(), "stale marker");
}
#[test]
fn empty_input_and_series_without_results() {
let window = RangeWindow {
+13 -100
View File
@@ -32,7 +32,7 @@ use datatypes::series::time::{EvalGrid, GridRange, TimeAxis, union_positions};
use datatypes::series::{Label, SeriesLabels};
use crate::go_sort;
use crate::grid_series::{GridSeriesBuilder, external};
use crate::grid_series::{GridSeriesBuilder, external, owned};
#[cfg(test)]
mod tests;
@@ -90,28 +90,17 @@ pub fn topk_at(
// positions, not the whole grid.
grid_points(schema, series)?;
let mut order: Vec<usize> = (0..series.len()).collect();
// The label pairs of a series are built once: a comparator that compared two metrics would
// rebuild both lists per comparison, and the groups read the same lists again.
let mut label_sets: Vec<Vec<(Arc<str>, Arc<str>)>> = series
.iter()
.map(|item| labels_with_name(&item.metric))
.collect();
order.sort_by(|&a, &b| label_sets[a].cmp(&label_sets[b]));
order.sort_by(|&a, &b| compare_labels(&series[a].metric, &series[b].metric));
let mut groups: HashMap<GroupKey, Vec<usize>> = HashMap::new();
for index in order {
groups
.entry(group_key_from(&label_sets[index], grouping))
.entry(group_key(&series[index].metric, grouping))
.or_default()
.push(index);
}
// The groups hold what they need, so the label lists are freed before the point walk.
label_sets.clear();
let mut selected: Vec<Vec<(usize, usize)>> = vec![Vec::new(); series.len()];
let mut candidates: Vec<(usize, usize, f64)> = Vec::new();
// The values of one point, for the partial selection below: ranking them there leaves the
// candidates in member order, which is what hands ties to the earlier label set.
let mut values: Vec<f64> = Vec::new();
for members in groups.values() {
// A point no member carries holds no candidate, so only the union of the members'
// positions is visited: a hole in the grid never reaches the ranking. Each member reads
@@ -133,41 +122,12 @@ pub fn topk_at(
candidates.push((member, row, value));
}
}
// Members are in label order, so a stable sort hands ties to the earlier label set.
candidates.sort_by(|a, b| rank(a.2, b.2, top));
let Some(k) = k_at(point)? else {
continue;
};
// Which candidates win is the result; the order they were found in is not. The ranking
// is a partial selection of the k best values rather than an ordering of the whole
// group, and it runs on a copy of the values so that the candidates keep the member
// order the ties are resolved in.
let k = k.min(candidates.len());
if k == 0 {
continue;
}
let boundary = (k < candidates.len()).then(|| {
values.clear();
values.extend(candidates.iter().map(|candidate| candidate.2));
values.select_nth_unstable_by(k, |a, b| rank(*a, *b, top));
values[k]
});
// The partition splits the run of values equal to its boundary, while ties go to the
// earlier label set: the ones equal to the boundary inside the front are the slots left
// for them, and the scan below fills those slots in member order, which is label order.
let mut tied = match boundary {
Some(boundary) => values[..k]
.iter()
.filter(|&&value| rank(value, boundary, top) == Ordering::Equal)
.count(),
None => 0,
};
for &(member, row, value) in &candidates {
if let Some(boundary) = boundary {
match rank(value, boundary, top) {
Ordering::Less => {}
Ordering::Equal if tied > 0 => tied -= 1,
_ => continue,
}
}
for &(member, row, _) in candidates.iter().take(k) {
selected[member].push((point, row));
}
}
@@ -258,9 +218,8 @@ pub fn limitk(
// A series belongs to one group, whose points are visited in order.
let mut cursor = series[index].fields.as_slice()[0].cursor();
for (point, row) in points {
// The sample is moved, not computed, so it goes straight into the result.
let sample = cursor.at(row).expect("the axis and the fields are aligned");
builder.push_refs(point, &[sample])?;
builder.push_samples(point, &[owned(sample)?])?;
}
result.extend(builder.finish(series[index].metric.clone())?);
}
@@ -281,7 +240,7 @@ pub fn limit_ratio(ratio: f64, schema: &SeriesSchema, series: &[Series]) -> Resu
let ratio = ratio.clamp(-1.0, 1.0);
let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone()))?;
let mut result = Vec::new();
let mut row: Vec<SampleRef<'_>> = Vec::with_capacity(schema.fields.len());
let mut row = Vec::with_capacity(schema.fields.len());
for item in series {
let offset = sample_offset(&item.metric);
let kept = if ratio >= 0.0 {
@@ -298,15 +257,14 @@ pub fn limit_ratio(ratio: f64, schema: &SeriesSchema, series: &[Series]) -> Resu
item.fields.iter().map(|field| field.cursor()).collect();
for (point, index) in item.time.positions() {
row.clear();
// The samples are moved, not computed, so the row is read and pushed as it stands.
for cursor in &mut cursors {
row.push(
row.push(owned(
cursor
.at(index)
.expect("the axis and the fields are aligned"),
);
)?);
}
builder.push_refs(point, &row)?;
builder.push_samples(point, &row)?;
}
result.extend(builder.finish(item.metric.clone())?);
}
@@ -580,8 +538,6 @@ const GO_NAN: f64 = f64::from_bits(0x7FF8_0000_0000_0001);
/// The order of `a` and `b` for `topk` (`top`) or `bottomk`; NaN ranks last for both.
fn rank(a: f64, b: f64, top: bool) -> Ordering {
#[cfg(test)]
calls::record_rank();
match (a.is_nan(), b.is_nan()) {
(true, true) => Ordering::Equal,
(true, false) => Ordering::Greater,
@@ -652,13 +608,8 @@ pub(crate) fn validate_topk_parameters(values: &[Option<f64>]) -> Result<()> {
type GroupKey = Vec<(Arc<str>, Arc<str>)>;
fn group_key(metric: &SeriesLabels, grouping: &Grouping) -> GroupKey {
group_key_from(&labels_with_name(metric), grouping)
}
/// The group `labels` belong to, for callers that already hold the series' label pairs.
fn group_key_from(labels: &[(Arc<str>, Arc<str>)], grouping: &Grouping) -> GroupKey {
let mut key: Vec<_> = labels
.iter()
let mut key: Vec<_> = labels_with_name(metric)
.into_iter()
.filter(|(name, value)| {
!value.is_empty()
&& match grouping {
@@ -666,7 +617,6 @@ fn group_key_from(labels: &[(Arc<str>, Arc<str>)], grouping: &Grouping) -> Group
Grouping::Without(names) => &**name != "__name__" && !names.contains(name),
}
})
.cloned()
.collect();
key.sort();
key
@@ -674,8 +624,6 @@ fn group_key_from(labels: &[(Arc<str>, Arc<str>)], grouping: &Grouping) -> Group
/// The label pairs of `metric` with `__name__` among them, sorted by name.
fn labels_with_name(metric: &SeriesLabels) -> Vec<(Arc<str>, Arc<str>)> {
#[cfg(test)]
calls::record_label_list();
let mut labels: Vec<_> = metric
.tags()
.iter()
@@ -806,38 +754,3 @@ fn into_value(sample: SampleRef<'_>) -> Result<Option<f64>> {
fn cursor_value(cursor: &mut RowCursor<'_>, row: usize) -> Result<Option<f64>> {
into_value(cursor.at(row).expect("the axis and the fields are aligned"))
}
/// How often this thread built a series' label list or compared two candidates, so a test can hold
/// `topk` to the work it should not do rather than to the result, which every shape produces.
/// Test-only: a decorated sort and a partial selection select the same candidates.
#[cfg(test)]
pub(crate) mod calls {
use std::cell::Cell;
thread_local! {
/// Label lists [`super::labels_with_name`] built on this thread.
static LABEL_LISTS: Cell<usize> = const { Cell::new(0) };
/// Candidate pairs [`super::rank`] looked at on this thread.
static RANKS: Cell<usize> = const { Cell::new(0) };
}
/// Records one label list.
pub(crate) fn record_label_list() {
LABEL_LISTS.with(|lists| lists.set(lists.get() + 1));
}
/// Records one comparison of two candidates.
pub(crate) fn record_rank() {
RANKS.with(|ranks| ranks.set(ranks.get() + 1));
}
/// The label lists built on this thread since the last call, which starts a new count.
pub(crate) fn take_label_lists() -> usize {
LABEL_LISTS.with(|lists| lists.replace(0))
}
/// The candidate comparisons made on this thread since the last call, which starts a new count.
pub(crate) fn take_ranks() -> usize {
RANKS.with(|ranks| ranks.replace(0))
}
}
+2 -144
View File
@@ -24,7 +24,7 @@ use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis};
use datatypes::series::{Label, SeriesLabels};
use super::{
Grouping, LabelMatcher, MatchOp, SortOrder, absent, absent_labels, calls, limit_ratio, limitk,
Grouping, LabelMatcher, MatchOp, SortOrder, absent, absent_labels, limit_ratio, limitk,
natural_less, sample_offset, scalar, sort_instant, topk,
};
use crate::golden::fold_pieces;
@@ -33,7 +33,7 @@ use crate::grid_series::reference::{
run_ranges, sparse_series,
};
use crate::grid_series::{GridSample, GridSeriesBuilder};
use crate::histogram_eval::float_histogram::{FloatHistogram, copies};
use crate::histogram_eval::float_histogram::FloatHistogram;
fn schema(points: u64, fields: usize) -> SeriesSchema {
SeriesSchema {
@@ -498,148 +498,6 @@ fn limit_operators_keep_the_start_timestamp_of_a_selected_sample() {
);
}
/// The selections that only move samples hand them from the field to the builder: no owned
/// histogram is built on the way, and the payload arrives as it was read.
#[test]
fn limit_operators_pass_histograms_through_without_an_owned_form() {
let histogram =
OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///");
let schema = schema(3, 1);
let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone())).unwrap();
builder
.push_refs(0, &[SampleRef::Histogram(histogram.view())])
.unwrap();
builder
.push_refs(2, &[SampleRef::Histogram(histogram.view())])
.unwrap();
let series = builder.finish(metric("__name__=a")).unwrap().unwrap();
copies::take_materialized();
let by_ratio = limit_ratio(1.0, &schema, std::slice::from_ref(&series)).unwrap();
let by_count = limitk(
1.0,
&Grouping::By(Vec::new()),
&schema,
std::slice::from_ref(&series),
)
.unwrap();
assert_eq!(
copies::take_materialized(),
0,
"a selection built an owned histogram of a sample it only moved"
);
// Both selections hand on the samples they kept, unchanged and in the same place.
let kept = |series: &Series| -> Vec<(usize, bool)> {
let samples = &series.fields.as_slice()[0];
series
.time
.positions()
.map(|(point, row)| {
(
point,
matches!(samples.sample(row), SampleRef::Histogram(_)),
)
})
.collect()
};
for kept_series in [&by_ratio, &by_count] {
assert_eq!(kept_series.len(), 1);
assert_eq!(kept(&kept_series[0]), [(0, true), (2, true)]);
}
}
/// `topk` builds one label list per series and ranks each point with a partial selection: the
/// comparisons it makes grow with the candidates rather than with their logarithm, which is what
/// ordering the whole group at every point cost.
#[test]
fn topk_ranks_a_point_with_a_partial_selection() {
let schema = schema(1, 1);
// Every series is one group, and the input arrives in reverse label order, so nothing about it
// is inherited from the order the members were built in.
let series: Vec<Series> = (0..256)
.rev()
.map(|index| {
let value = if index < 16 { 10.0 } else { 1.0 };
build(
&schema,
metric(&format!("__name__=s{index:03}")),
&vec![vec![Some(value)]],
)
.unwrap()
})
.collect();
calls::take_label_lists();
calls::take_ranks();
let kept = topk(8.0, true, &Grouping::By(Vec::new()), &schema, &series).unwrap();
let lists = calls::take_label_lists();
let comparisons = calls::take_ranks();
let mut kept = rendered(&kept);
kept.sort();
assert_eq!(
kept,
[
"__name__=s000",
"__name__=s001",
"__name__=s002",
"__name__=s003",
"__name__=s004",
"__name__=s005",
"__name__=s006",
"__name__=s007"
],
);
assert_eq!(
lists,
series.len(),
"topk rebuilt a series' label list instead of reading the one it built"
);
// Ordering the whole group at the point would take the candidates' logarithm per candidate;
// the partial selection takes a small constant.
assert!(
comparisons <= 4 * series.len(),
"topk compared {comparisons} candidate pairs to select 8 of {}",
series.len()
);
}
/// Ties go to the label set that sorts first, whatever order the input arrived in: the value the
/// partial selection stops at falls inside a run of equal values, and the run is resolved by label
/// order rather than by the order the ranking left the group in.
#[test]
fn topk_breaks_ties_by_label_order() {
let names: Vec<String> = (0..40)
.map(|index| format!("__name__=s{index:02}"))
.collect();
// The ten series that sort first carry the top value, and the input arrives in reverse label
// order, so a tie can only be resolved by the labels.
let values: Vec<(&str, Option<f64>)> = names
.iter()
.rev()
.map(|name| {
let value = if name.as_str() < "__name__=s10" {
7.0
} else {
3.0
};
(name.as_str(), Some(value))
})
.collect();
let (schema, series) = instant(&values);
let kept = topk(3.0, true, &Grouping::By(Vec::new()), &schema, &series).unwrap();
let mut kept = rendered(&kept);
kept.sort();
assert_eq!(kept, ["__name__=s00", "__name__=s01", "__name__=s02"]);
// `bottomk` ties the same way, and its run of equal values holds the whole group.
let kept = topk(3.0, false, &Grouping::By(Vec::new()), &schema, &series).unwrap();
let mut kept = rendered(&kept);
kept.sort();
assert_eq!(kept, ["__name__=s10", "__name__=s11", "__name__=s12"]);
}
fn grouping(kind: &str, labels: &str) -> Grouping {
match kind {
"without" => Grouping::Without(names(labels)),
+7 -142
View File
@@ -70,33 +70,6 @@ pub(crate) fn visited_pairs() -> usize {
VISITED_PAIRS.with(std::cell::Cell::get)
}
// How many matches the binary kernel recorded in this thread's match set.
//
// Test-only, and per thread: whether a field needs the set at all is what decides the per-sample
// work of the default one-to-one walk, and that is not visible in the result. A thread of its own
// keeps one test's matches out of another test's figure, and the counter is compiled out of the
// kernel otherwise.
#[cfg(test)]
thread_local! {
static MATCH_RECORDS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// Resets the match-set counter of this thread to zero.
#[cfg(test)]
pub(crate) fn reset_match_records() {
MATCH_RECORDS.with(|recorded| recorded.set(0));
}
/// The matches the binary kernel has recorded in this thread's match set since the last
/// [`reset_match_records`].
///
/// A match is recorded once per row the operator keeps, so the count is what the walk does rather
/// than what its operands' sizes imply it would.
#[cfg(test)]
pub(crate) fn match_records() -> usize {
MATCH_RECORDS.with(std::cell::Cell::get)
}
/// A PromQL binary operator between two instant vectors.
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Hash)]
pub enum BinaryOp {
@@ -895,21 +868,6 @@ fn binary_field(
}
let drop_name = !matching.op.is_comparison() || matching.return_bool;
// One-to-one: a match group may be matched once per point; Prometheus counts a filtered
// comparison as a match too, but not an operation that failed. Filling the one side consults the
// same set. A group whose many side holds one series is matched at a point by one pairing only --
// a second row there would have to come from a second one-side series, and two members of one
// group present at one point is the duplicate `overlap` already reports -- so the set is built
// only for the fields that can hold two matches of one group at one point, and the walk of the
// default one-to-one shape records nothing at all.
let matched = fill.lhs.is_some()
|| (matching.cardinality == Cardinality::OneToOne
&& shared_match_group(
many,
&matching.labels,
metadata.tables.saturating_add(metadata.pairing),
budget.as_deref_mut(),
)?);
let mut results = Results::default();
let mut pairs = Pairs {
matching,
@@ -917,7 +875,11 @@ fn binary_field(
annotations,
group_keys: Vec::new(),
group_ids: HashMap::new(),
matched: matched.then(HashSet::new),
// One-to-one: a match group may be matched once per point; Prometheus counts a filtered
// comparison as a match too, but not an operation that failed. Filling the many side
// consults the same set.
matched: (matching.cardinality == Cardinality::OneToOne || fill.lhs.is_some())
.then(HashSet::new),
implicit_many_to_one: None,
};
for (many_index, many_series) in many.series.iter().enumerate() {
@@ -1147,96 +1109,6 @@ fn binary_field(
Ok(results)
}
/// Whether two series of `operand` share a match group.
///
/// The keys are compared by a hash of the labels they keep: two series that share a group hash
/// equal, and two that differ may still collide -- an answer of `true` for a group that no two
/// series share only makes the caller build the set a shared group would need. One `u64` per series
/// is sorted in place and dropped with the check, so a field of single-member groups builds no key
/// and keeps none, whatever the length of the grid.
fn shared_match_group(
operand: &Operand<'_>,
labels: &MatchingLabels,
base: usize,
mut budget: Option<&mut Budget<'_>>,
) -> Result<bool> {
if operand.series.len() < 2 {
return Ok(false);
}
if let Some(budget) = budget.as_deref_mut() {
// One hash per series, entered beside the field's own tables before the list is allocated.
budget.phase(
base.saturating_add(
operand
.series
.len()
.saturating_mul(std::mem::size_of::<u64>()),
),
)?;
}
let mut hashes: Vec<u64> = Vec::with_capacity(operand.series.len());
hashes.extend(
operand
.series
.iter()
.map(|series| match_group_hash(&series.metric, labels)),
);
hashes.sort_unstable();
let shared = hashes.windows(2).any(|pair| pair[0] == pair[1]);
// The list is dropped with the check, so the field's own phase figure is live from here on.
if let Some(budget) = budget {
budget.phase(base)?;
}
Ok(shared)
}
/// The FNV-1a offset basis and prime, which hash one label's name and value in the same steps
/// whatever hasher the tables of a field happen to use.
const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
/// The hash of the labels [`group_key`] would keep for `metric`.
///
/// Each kept label hashes its name and value as one string, and the labels are summed with
/// wrapping addition, so the order the label set stores them in cannot change the answer: two
/// series of one match group hash equal however they are laid out.
fn match_group_hash(metric: &SeriesLabels, labels: &MatchingLabels) -> u64 {
let label = |name: &str, value: &str| {
let mut hash = FNV_OFFSET_BASIS;
for byte in name
.as_bytes()
.iter()
.chain(&[0xff])
.chain(value.as_bytes())
{
hash ^= u64::from(*byte);
hash = hash.wrapping_mul(FNV_PRIME);
}
hash
};
let mut hash = 0u64;
match labels {
MatchingLabels::On(names) => {
for name in names {
// An empty value is absent, as in `group_key`.
if let Some(value) = metric.get(name)
&& !value.is_empty()
{
hash = hash.wrapping_add(label(name, value));
}
}
}
MatchingLabels::Ignoring(ignored) => {
for tag in metric.tags() {
if !tag.value.is_empty() && !ignored.contains(&tag.name) {
hash = hash.wrapping_add(label(&tag.name, &tag.value));
}
}
}
}
hash
}
/// The matched pairs of one field and the state Prometheus checks them against.
///
/// The results are filed beside it, not in it: a pairing's open row borrows the table it files into
@@ -1248,10 +1120,8 @@ struct Pairs<'a> {
group_keys: Vec<GroupKey>,
group_ids: HashMap<GroupKey, usize>,
/// The match groups an operation matched at a point, kept where Prometheus consults them:
/// one-to-one matching, and filling the one side. A filtered comparison counts as a match, an
/// operation that failed does not. Kept only for the fields that need it: one whose groups hold
/// one many-side series cannot match a group twice at a point, and one filling neither side never
/// asks.
/// one-to-one matching, and filling the many side. A filtered comparison counts as a match,
/// an operation that failed does not.
matched: Option<HashSet<(usize, usize)>>,
implicit_many_to_one: Option<(usize, usize)>,
}
@@ -1325,11 +1195,6 @@ impl Pairs<'_> {
// One-to-one: a match group may be matched once per point; Prometheus counts a filtered
// comparison as a match too, but not an operation that failed.
if let Some(matched) = &mut self.matched {
// Matches are what the set is for, and it exists only for the fields that can hold two
// of one group at one point: the test-only counter reports that the walk of a
// single-member group records nothing.
#[cfg(test)]
MATCH_RECORDS.with(|recorded| recorded.set(recorded.get() + 1));
// The match set is the one container this walk grows point by point, so it is charged by
// the block it takes when it outgrows what it holds -- before the insert allocates it, and
// never per point of a set that has room.
+2 -197
View File
@@ -27,9 +27,8 @@ use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use super::{
BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, RESULT_ROW_BYTES, SetOp,
binary_vectors, binary_vectors_with_reservation, match_records, reset_match_records,
set_vectors, set_vectors_with_reservation,
BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, SetOp, binary_vectors,
binary_vectors_with_reservation, set_vectors, set_vectors_with_reservation,
};
use crate::grid_series::reference::{
Dense, OwnedHistogram, Points, assert_histogram_close, assert_result_eq, golden_cases,
@@ -1200,163 +1199,6 @@ fn colliding_reset_hints_do_not_count_as_a_match() {
}
}
/// A group whose many side holds one series can be matched at a point by one pairing only: a second
/// row for that point would have to come from a second one-side series, and two members of one group
/// present at one point is the duplicate the one side's own scan reports. So the default one-to-one
/// shape records no match and builds no set, and the pairing still answers every point.
#[test]
fn a_single_series_group_records_no_match() {
const POINTS: usize = 1_000;
let grid = budget_grid(POINTS);
let schema = Arc::new(schema(1, &grid));
let at: Vec<usize> = (0..POINTS).collect();
// One series of the many side and one of the one side, in the same match group.
let lhs = vec![budget_operand(&schema, "a", &at)];
let rhs = vec![budget_operand(&schema, "b", &at)];
let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new()));
reset_match_records();
let output =
binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("the one pair matches");
assert_eq!(
match_records(),
0,
"a group of one many-side series recorded a match"
);
// Both operands hold the point's own position, so the sum doubles it at every point.
assert_eq!(output.len(), 1, "{output:?}");
let mut cursor = output[0].fields.as_slice()[0].cursor();
let mut rows = 0;
for (point, row) in output[0].time.positions() {
match cursor.at(row).expect("the axis and the fields are aligned") {
SampleRef::Float(value) => assert_eq!(value, 2.0 * point as f64, "point {point}"),
_ => panic!("`+` of two floats keeps a float"),
}
rows += 1;
}
assert_eq!(rows, POINTS, "the pair answers every point");
}
/// A group of two many-side series can be matched twice at one point, so the set is built and the
/// second match of the group at that point is the duplicate Prometheus reports.
#[test]
fn a_group_of_two_many_series_records_its_matches() {
let grid = budget_grid(1);
let schema = Arc::new(schema(1, &grid));
// Two left-hand series in one match group and one right-hand series they both pair with.
let lhs = [
jobbed_operand(&schema, "a", "x", &[0]),
jobbed_operand(&schema, "b", "x", &[0]),
];
let rhs = [jobbed_operand(&schema, "c", "x", &[0])];
let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new()));
reset_match_records();
let error = binary_vectors(&matching, &schema, &lhs, &schema, &rhs)
.unwrap_err()
.to_string();
assert!(
error.contains("many-to-one matching must be explicit"),
"{error}"
);
assert!(
match_records() >= 2,
"the second match of the group was not recorded: {}",
match_records()
);
}
/// Two one-side series of a group that are never present at the same point are no duplicate: each
/// point is matched by one pairing, which is what a group of one many-side series allows.
#[test]
fn a_group_matched_at_different_points_is_no_duplicate() {
const POINTS: usize = 64;
let grid = budget_grid(POINTS);
let schema = Arc::new(schema(1, &grid));
let operand = |name: &str, instance: &str, value: f64, at: &[usize]| -> Series {
let labels = Labels::from([
("__name__".to_string(), name.to_string()),
("job".to_string(), "x".to_string()),
("instance".to_string(), instance.to_string()),
]);
let points: Points = at
.iter()
.map(|&point| (point, f64::to_bits(value)))
.collect();
sparse_series(&schema, &labels, &points).expect("the operand holds a value")
};
let even: Vec<usize> = (0..POINTS).step_by(2).collect();
let odd: Vec<usize> = (1..POINTS).step_by(2).collect();
// One many-side series holding every point, and two one-side series that split the grid between
// them: the group is matched at every point, never twice at one.
let lhs = [operand("a", "many", 0.5, &(0..POINTS).collect::<Vec<_>>())];
let rhs = [
operand("b", "even", 1.0, &even),
operand("c", "odd", 100.0, &odd),
];
let matching = budget_matching(BinaryOp::Add, MatchingLabels::On(vec![Arc::from("job")]));
reset_match_records();
let output =
binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("no point is matched twice");
assert_eq!(
match_records(),
0,
"a group of one many-side series recorded a match"
);
// One-to-one matching names the result after the many side and the matching labels, so both
// pairings file under one label set: the series holds the even points a `b` kept and the odd ones
// a `c` kept.
assert_eq!(output.len(), 1, "{output:?}");
let mut cursor = output[0].fields.as_slice()[0].cursor();
let mut rows = 0;
for (point, row) in output[0].time.positions() {
let expected = if point % 2 == 0 { 1.5 } else { 100.5 };
match cursor.at(row).expect("the axis and the fields are aligned") {
SampleRef::Float(value) => assert_eq!(value, expected, "point {point}"),
_ => panic!("`+` of two floats keeps a float"),
}
rows += 1;
}
assert_eq!(rows, POINTS, "the two pairings cover the grid between them");
}
/// A fill value of the one side is answered from the matches the walk recorded, so a pairing that
/// holds at a point keeps it and only the points it left unmatched are filled -- the query of an
/// arbitrary point of a group, not a run of them.
#[test]
fn a_fill_consults_the_match_set_point_by_point() {
let grid = budget_grid(2);
let schema = Arc::new(schema(1, &grid));
let lhs = [jobbed_operand(&schema, "a", "x", &[0])];
let rhs = [jobbed_operand(&schema, "b", "x", &[0, 1])];
let matching = BinaryMatching {
fill: FillValues {
lhs: Some(1_000.0),
rhs: None,
},
..budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new()))
};
reset_match_records();
let output =
binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("the operands are readable");
// Filling the one side is what asks the set whether a point was matched, so the field builds it:
// the pair at point 0 records its match, and so does the row point 1 is filled with.
assert!(
match_records() > 0,
"filling the one side did not record the match it consults"
);
assert_eq!(output.len(), 1, "{output:?}");
let mut cursor = output[0].fields.as_slice()[0].cursor();
let mut kept = Vec::new();
for (point, row) in output[0].time.positions() {
match cursor.at(row).expect("the axis and the fields are aligned") {
// Point 0 is the pair's own sum; at point 1 the fill stands in for the sample the left
// side has at no point, and the operator is applied to it as it would be to a sample.
SampleRef::Float(value) => kept.push((point, value)),
_ => panic!("`+` of two floats keeps a float"),
}
}
assert_eq!(kept, [(0, 0.0), (1, 1_001.0)], "{output:?}");
}
/// A pair whose operation fails is not a match, so `fill_left` still fills for the right-hand
/// sample of that pair; the fixtures hold floats only.
#[test]
@@ -2615,43 +2457,6 @@ fn a_dense_result_is_refused_where_a_sparse_one_is_admitted() {
);
}
/// A single field's rows reach the builder as they were filed: no table of triples is keyed by the
/// label set the rows already sit under, so what the merge adds to the rows the field holds is the
/// columns it writes them into, and a second copy of every sample is not part of the figure.
#[test]
fn a_single_field_holds_no_second_copy_of_its_rows() {
const POINTS: usize = 8_192;
let grid = budget_grid(POINTS);
let schema = Arc::new(schema(1, &grid));
let at: Vec<usize> = (0..POINTS).collect();
let lhs = vec![budget_operand(&schema, "a", &at)];
let rhs = vec![budget_operand(&schema, "b", &at)];
let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new()));
let (output, seen) = budgeted(&matching, &schema, &lhs, &schema, &rhs, usize::MAX);
let output = output.expect("the one pair matches");
assert_eq!(output.len(), 1, "{output:?}");
assert_eq!(
run_ranges(&output[0]).len(),
1,
"the pair keeps one run of the whole grid"
);
let peak = *seen
.iter()
.max()
.expect("the operation asks for its workspace");
assert!(peak > 0, "the operation asks for its workspace");
// What the operation holds once the field's tables are released is the rows it filed and the
// builder's own buffers: a merge that copied every row into a triple of its own table would hold
// a second copy of every sample beside them.
let floor = retained_total(&seen);
let rows = POINTS * RESULT_ROW_BYTES;
assert!(floor >= rows, "floor {floor}, rows {rows}");
assert!(
floor <= rows + 8 * 1024,
"floor {floor} for {rows} row bytes: the merge holds a second copy of every sample"
);
}
/// A field that meets its first histogram after a float backfills the index column of the rows
/// before it, and the budget covers the column all the same.
#[test]
-1
View File
@@ -72,7 +72,6 @@ session.workspace = true
snafu.workspace = true
sql.workspace = true
sqlparser.workspace = true
stacker.workspace = true
store-api.workspace = true
substrait.workspace = true
table.workspace = true
-30
View File
@@ -126,36 +126,6 @@ lazy_static! {
&["result"]
)
.unwrap();
/// Statements the native PromQL route handed to the existing planner, by the cause it recorded.
///
/// A declined statement is not an error and not a different answer: the existing planner
/// computes the same rows, so a fallback is only visible as a slower query. This counter and
/// the log line beside it are what make its rate observable per cause.
pub static ref NATIVE_PROMQL_DECLINED_TOTAL: IntCounterVec = register_int_counter_vec!(
"greptime_query_native_promql_declined_total",
"native PromQL expressions planned by the existing planner, by cause",
&["reason"]
)
.unwrap();
/// Statements whose metadata drop the native PromQL planner collected for the scan's key
/// columns alone, by reason.
///
/// The canonical pass collects the pieces of one label set before a consumer that reads whole
/// series, and it decides that from the labels a tree reports and from the key columns the scan
/// below them keys its series by. The second half is what one pair of stored series needs: a
/// primary-key column that holds `NULL` in one series and `''` in another decodes to an absent
/// and to an empty label, and a name-dropping kernel drops the empty one with the metadata
/// labels, leaving both under one label set. A drop the labels alone prove is collected for that
/// pair, and this counter is what makes the statements that pay for it observable.
pub static ref DUPLICATE_LABELSET_CHECK_SKIPPED_TOTAL: IntCounterVec =
register_int_counter_vec!(
"greptime_query_duplicate_labelset_check_skipped_total",
"native PromQL statements whose metadata drop was collected for their key columns alone, by reason",
&["reason"]
)
.unwrap();
}
/// A stream to call the callback once a RecordBatch stream is done.
+6 -112
View File
@@ -22,10 +22,7 @@ pub use native::{NATIVE_PROMQL_EXTENSION_KEY, discovers_metric_name, native_prom
mod set_operator;
use std::collections::{BTreeSet, HashSet, VecDeque};
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use arrow::datatypes::IntervalDayTime;
use async_recursion::async_recursion;
@@ -220,45 +217,6 @@ struct PromPlannerContext {
range_fold_offset: Option<Millisecond>,
}
/// Stack one planning descent level needs below it before the recursion grows its stack.
///
/// The planner recurses once per operator and the descent is a chain of nested `poll`s on boxed
/// futures, so each level's frame is charged to the stack of the level above it. A debug build's
/// frame is large: a chain of eight constant multiplications under a propagated matcher needs more
/// than the 2 MiB a worker thread has, and there is no depth limit on the expressions this route
/// accepts. The recursion grows its stack instead, at the same reserve the native runtime's own
/// walks use.
const PLANNER_STACK_RED_ZONE: usize = 512 * 1024;
/// Stack a growth adds when the reserve above is spent. It holds several levels of the descent.
const PLANNER_STACK_GROWTH: usize = 4 * 1024 * 1024;
/// Polls one planning descent on a stack that grows on demand.
///
/// `stacker::maybe_grow` needs a synchronous callee to run on the segment it switches to, which an
/// `async` body cannot be, so the growth wraps a `poll` instead: the segment stays live for that
/// whole poll, every nested level the poll reaches included, and each nested level wraps its own
/// poll in turn. This is the same reserve-and-grow rule `promql::native::execute` applies to the
/// runtime node walks, which recurse synchronously.
struct GrownStack<F>(Pin<Box<F>>);
impl<F: Future> GrownStack<F> {
fn new(future: F) -> Self {
Self(Box::pin(future))
}
}
impl<F: Future> Future for GrownStack<F> {
type Output = F::Output;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let mut inner = self.get_mut().0.as_mut();
stacker::maybe_grow(PLANNER_STACK_RED_ZONE, PLANNER_STACK_GROWTH, || {
inner.as_mut().poll(cx)
})
}
}
/// Result labels a vector-vector binary operation derives from its matching modifier, projected
/// from the operand each label belongs to.
#[derive(Debug)]
@@ -346,14 +304,6 @@ pub struct PromPlanner {
/// The engine-wide `QueryOptions::experimental_native_promql` is applied while `QueryEngineState`
/// is built; this planner reads the per-query request extension instead.
native_fused: bool,
/// Whether the binary node being planned may still offer its own subtree to the native route.
///
/// An offer compiles a whole tree, so it is made where it covers the most: the statement root,
/// and the topmost binary below a node this route does not answer. Once a binary node has spent
/// the offer, the operands below it are planned by the existing path without offering
/// themselves again - re-offering each operand of a refused chain of n operators compiles its
/// subtrees again and costs O(n^2) passes instead of O(n).
native_binary_offer: bool,
}
type BinaryFieldPair<'a> = (&'a String, &'a String);
@@ -386,9 +336,6 @@ impl PromPlanner {
ctx: PromPlannerContext::from_eval_stmt(stmt)?,
promql_annotations,
native_fused,
// The statement's own expression is offered below before anything else is planned, so
// the first binary node reached still holds the offer a wrapper would have spent.
native_binary_offer: true,
};
// The whole statement is offered to the native compiler first, because that compiler folds
@@ -433,32 +380,6 @@ impl PromPlanner {
prom_expr: &PromExpr,
timestamp_fn: bool,
query_engine_state: &QueryEngineState,
) -> Result<LogicalPlan> {
// Every operator of the existing expression planning descends through here, so this is
// where that recursion grows its stack instead of spending the caller's, below the reserve
// [`PLANNER_STACK_RED_ZONE`] names. Two limits come with it:
//
// - The native fused-series route is no part of this recursion. Its own composition
// recursion grows at its own boundary (`PromPlanner::try_plan_native` in `native.rs`),
// not here.
// - The synchronous walks around it are not covered at all: the literal construction
// (`try_build_literal_expr`) and the logical row-key walk
// (`collect_row_key_tag_columns_from_plan`) in this file, and the topology walk
// (`native_raw_plan_bounds_region_reads`), the root-parenthesis traversal
// (`root_subquery`), the input-order dependency walk (`depends_on_input_order`), the
// selector collection (`collect_selectors_with_set_operations`) and the native root
// subquery compilation in `native.rs`, recurse synchronously on the stack their caller
// has. A deep enough expression or plan still costs them that stack.
let step = self.prom_expr_to_plan_step(prom_expr, timestamp_fn, query_engine_state);
GrownStack::new(step).await
}
#[async_recursion]
async fn prom_expr_to_plan_step(
&mut self,
prom_expr: &PromExpr,
timestamp_fn: bool,
query_engine_state: &QueryEngineState,
) -> Result<LogicalPlan> {
// An anchored range call is step-invariant: evaluate it once, at the start of the
// evaluation, and report its result at every step; see
@@ -480,18 +401,8 @@ impl PromPlanner {
.await?
}
PromExpr::Binary(expr) => {
// One native offer per binary subtree: the offer at a node covers every node below
// it, because the compiler reads the whole tree at once. It is spent here, by the
// node that makes it, and handed back once the subtree is planned, so a refused
// chain of n operators makes one offer per position instead of one per operator
// and per subtree below it - O(n) instead of O(n^2). A wrapper this route cannot
// answer leaves the offer to the binary that sits under it.
let offer = std::mem::take(&mut self.native_binary_offer);
let plan = self
.prom_binary_expr_to_plan(query_engine_state, expr, offer)
.await;
self.native_binary_offer = offer;
plan?
self.prom_binary_expr_to_plan(query_engine_state, expr)
.await?
}
PromExpr::Paren(ParenExpr { expr }) => {
self.prom_expr_to_plan_inner(expr, timestamp_fn, query_engine_state)
@@ -1275,13 +1186,10 @@ impl PromPlanner {
})
}
/// Plans `binary_expr`; `offer` says whether this node may still offer its subtree to the
/// native route, which the caller spends for the subtree it is about to plan.
async fn prom_binary_expr_to_plan(
&mut self,
query_engine_state: &QueryEngineState,
binary_expr: &PromBinaryExpr,
offer: bool,
) -> Result<LogicalPlan> {
// promql-parser accepts fill modifiers, but Greptime does not implement the
// required outer joins and missing-value substitution. Reject them before the
@@ -1298,7 +1206,6 @@ impl PromPlanner {
// The same whole native binary the statement root was offered can sit below a
// legacy-only wrapper. A nested retry is valid only on the statement's own grid.
if self.native_fused
&& offer
&& self.ctx.start == self.ctx.stmt_start
&& self.ctx.end == self.ctx.stmt_end
{
@@ -1553,32 +1460,19 @@ impl PromPlanner {
// A copied matcher belongs to the scan, not to the operand's identity:
// `absent()` turns `selector_matcher` into the labels it reports, so the
// re-planned context keeps the matchers the operand was written with.
//
// A rewritten operand is an expression this node's offer never compiled: that
// offer read the statement as it was written, and the matcher reached the scan
// afterwards. The re-plan is therefore handed the offer, which the rewritten
// operand then spends on its own subtree exactly as this node spent it - so a
// chain below the rewrite still costs one offer, not one per operator and not
// one per operator and subtree below it.
if rewritten.lhs.as_ref() != lhs.as_ref() {
let selectors = std::mem::take(&mut left_context.selector_matcher);
let offer = std::mem::replace(&mut self.native_binary_offer, true);
let replanned = self
left_input = self
.prom_expr_to_plan(&rewritten.lhs, query_engine_state)
.await;
self.native_binary_offer = offer;
left_input = replanned?;
.await?;
left_context = self.ctx.clone();
left_context.selector_matcher = selectors;
}
if rewritten.rhs.as_ref() != rhs.as_ref() {
let selectors = std::mem::take(&mut right_context.selector_matcher);
let offer = std::mem::replace(&mut self.native_binary_offer, true);
let replanned = self
right_input = self
.prom_expr_to_plan(&rewritten.rhs, query_engine_state)
.await;
self.native_binary_offer = offer;
right_input = replanned?;
.await?;
right_context = self.ctx.clone();
right_context.selector_matcher = selectors;
}
+98 -314
View File
@@ -39,7 +39,6 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE, MITO2_ENGINE};
use common_query::native_histogram::native_histogram_value_type;
use common_query::prelude::OTLP_AGGREGATION_TEMPORALITY_LABEL;
use common_query::series_frame::raw_series_frame_schema_with_ordering;
use common_telemetry::debug;
use datafusion::arrow::datatypes::DataType as ArrowDataType;
use datafusion::common::{DFSchema, DFSchemaRef};
use datafusion::error::DataFusionError;
@@ -55,7 +54,7 @@ use datatypes::series::time::EvalGrid;
use futures::future::BoxFuture;
use promql::extension_plan::{
FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec, InputLocality,
RawSeriesSource, input_locality, keys_survive_metadata_drop, matching_operands_drop_key,
RawSeriesSource, input_locality, matching_operands_drop_key,
};
use promql::native::plan::{
AggregateKind, BoundAggregate, BoundInput, BoundVectorOp, NativeExpr, NativeNode, NativeTree,
@@ -65,7 +64,7 @@ use promql::native::plan::{
use promql::native::segment::{
self, NATIVE_ORDERING, canonical_boundary_stages, canonical_local_prefix,
contains_canonical_merge, contains_global_consumer, groups_by_metric_name,
region_local_segment, segment_tree, with_row_name_column,
merge_answers_histogram, region_local_segment, segment_tree, with_row_name_column,
};
// Kept reachable here because this crate reads them through this module's own path: the dist
// planner's commutativity check asks whether a fused node is per-series work, and this crate's
@@ -88,9 +87,7 @@ use crate::promql::native_compile::{
InfoInput, bind_evaluated_binary, compile_selectors_with_grid, compile_statement_with_grid,
info_input, statement_grid, subquery_child_grid,
};
use crate::promql::planner::{
DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, GrownStack, PromPlanner,
};
use crate::promql::planner::{DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, PromPlanner};
use crate::query_engine::QueryEngineState;
/// The query option that turns the native fused-series route on for one query.
@@ -186,13 +183,6 @@ type PreparedInput = (
// `split_series`: whether that partition splits one series across regions, which a bare scan
// cannot reconcile. A partition on tags keeps every series whole in one region.
bool,
// `keys_survive`: whether the resolved table keys its series by columns that always decode a
// value, read from its own primary key through `keys_survive_metadata_drop`. It is the half of
// the metadata drop's proof a tree cannot answer about itself: the drop removes every empty
// label value with the name, so a column holding `NULL` in one series and `''` in another
// leaves both under one label set. The canonical pass collects the drop when this is `false`,
// and `SKIPPED_LABELSET_CHECK_NULLABLE_KEY` counts the statements collected for that reason.
bool,
);
/// A fully placed native operand and the evaluated layout its frame carries.
@@ -230,86 +220,6 @@ fn native_planned(
}
}
thread_local! {
/// The cause the native route recorded for the expression it is about to refuse.
///
/// A refusal is answered by four different walks - the compiler binding, the scan preparation,
/// the output-schema derivation and the segmentation - and most of those sites are plain
/// functions that can only answer `None`. The cause therefore travels beside the `None` rather
/// than through it: a refusal records itself here, and the offer that receives the `None`
/// reports it. One statement is planned on one thread, and taking the record clears it.
static NATIVE_DECLINE: std::cell::Cell<Option<&'static str>> =
const { std::cell::Cell::new(None) };
/// How many native attempts this thread made, one per whole-expression offer.
///
/// An attempt compiles the whole tree below it, so this count times the statement's size bounds
/// the work planning it costs. A test uses it to hold a refused chain to one attempt per
/// position instead of one per position and per operator below it, which is O(n^2).
#[cfg(test)]
static NATIVE_ATTEMPTS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
/// The cause the route last reported for a refused offer.
///
/// The report is a log line and a counter, neither of which a test can read back; this holds
/// the same cause the two of them carry.
#[cfg(test)]
static REPORTED_DECLINE: std::cell::Cell<Option<&'static str>> =
const { std::cell::Cell::new(None) };
/// The reason this thread's statement had its metadata drop collected for its key columns.
///
/// The canonical pass decides that while it rewrites a tree, and the counter that reports it is
/// taken once per statement by the offer that answers the plan - so the reason travels beside
/// the tree the way a refusal travels beside the `None` it answers.
static SKIPPED_LABELSET_CHECK: std::cell::Cell<Option<&'static str>> =
const { std::cell::Cell::new(None) };
/// The reason the counter last reported for a drop collected over a nullable key.
///
/// The counter is not something a test can read back; this holds the same reason it carries.
#[cfg(test)]
static REPORTED_SKIPPED_LABELSET_CHECK: std::cell::Cell<Option<&'static str>> =
const { std::cell::Cell::new(None) };
}
/// The cause reported when a refusal was answered without recording one.
const NATIVE_DECLINE_UNRECORDED: &str = "cause_not_recorded";
/// The reason a statement's metadata drop was collected: the scan that answers it keys its series by
/// a column that can hold `NULL` in one series and `''` in another, so the labels alone cannot prove
/// the drop tells two of its series apart.
const SKIPPED_LABELSET_CHECK_NULLABLE_KEY: &str = "nullable_key";
/// Answers the `None` that hands this expression to the existing planner, recording why.
///
/// Declining is the normal answer to a shape this route does not answer: the existing planner
/// computes the same rows, so the site itself stays quiet. What it contributes is the cause, which
/// the offer that receives the `None` reports once.
fn declined<T>(reason: &'static str) -> Option<T> {
NATIVE_DECLINE.with(|record| record.set(Some(reason)));
None
}
/// Takes the cause the last refusal recorded, if it recorded one.
fn take_native_decline() -> Option<&'static str> {
NATIVE_DECLINE.with(|record| record.replace(None))
}
/// Clears the attempt count and answers how many attempts this thread made since the last call.
#[cfg(test)]
pub(crate) fn take_native_attempts() -> usize {
NATIVE_ATTEMPTS.with(|attempts| attempts.replace(0))
}
/// Takes the cause the route last reported for a refused offer.
#[cfg(test)]
pub(crate) fn take_reported_decline() -> Option<&'static str> {
REPORTED_DECLINE.with(|reported| reported.replace(None))
}
/// Takes the reason the route last reported for a drop collected over a nullable key.
#[cfg(test)]
pub(crate) fn take_reported_skipped_labelset_check() -> Option<&'static str> {
REPORTED_SKIPPED_LABELSET_CHECK.with(|reported| reported.replace(None))
}
impl PromPlanner {
/// Builds the scan of the one logical table the current context is bound to. The caller has
/// already preprocessed this selector's matchers and set the context to its table.
@@ -326,14 +236,14 @@ impl PromPlanner {
self.ctx.table_name.clone(),
self.ctx.time_index_column.clone(),
) else {
return Ok(declined("selector_without_table_or_time_index"));
return Ok(None);
};
let tags = self.ctx.tag_columns.clone();
let fields = self.ctx.field_columns.clone();
// A `__field__` matcher narrows the field set in a projection the existing planner builds, and the
// raw series scan below this node has no equivalent: it reports every field the scan carries.
if self.ctx.field_column_matcher.is_some() || fields.is_empty() {
return Ok(declined("field_selector_without_raw_series_equivalent"));
return Ok(None);
}
// Delta samples are increments: the existing planner sums them for `increase` and
// divides by the range for `rate`, while the native tree assumes cumulative samples.
@@ -343,7 +253,7 @@ impl PromPlanner {
.any(|tag| tag == OTLP_AGGREGATION_TEMPORALITY_LABEL)
&& under_rate_or_increase
{
return Ok(declined("delta_temporality_under_rate_or_increase"));
return Ok(None);
}
// The selector's own table scan, built by the same method the existing path uses:
// the physical planner turns it into the region scan the node reads, and building a
@@ -372,7 +282,7 @@ impl PromPlanner {
.map(|field| source_schema.index_of(field).ok())
.collect::<Option<Vec<_>>>()
else {
return Ok(declined("field_missing_from_source_schema"));
return Ok(None);
};
let original_projection = scan.projection.as_ref();
let selected_indices = original_projection
@@ -396,7 +306,7 @@ impl PromPlanner {
}
None => match &input {
LogicalPlan::TableScan(scan) => scan.clone(),
_ => return Ok(declined("selector_plan_is_not_a_table_scan")),
_ => return Ok(None),
},
};
let table = self.table_from_source(&scan.source)?;
@@ -405,18 +315,6 @@ impl PromPlanner {
// only collects the pieces its own producer holds, so the caller needs this fact to tell a whole-query
// label-set answer from one region's share of it.
let partitioned = table_meta.partition_column_names().next().is_some();
// A kernel that drops the metadata labels drops every label whose value is empty with them,
// so two stored series whose key columns hold `NULL` and `''` leave it as one identity. This
// is the fact the drop's proof needs beside the labels - the same rule the physical
// per-series scan reads to decide whether a node may stream - and the canonical pass
// collects the drop when it does not hold.
let keys_survive = keys_survive_metadata_drop(
tags.iter().map(String::as_str),
table_meta
.primary_key_indices
.iter()
.map(|index| &table_meta.schema.column_schemas()[*index]),
);
// An engine without a raw-series scanner has no frame to give this node, and by the time the physical
// planner asks, the logical plan is already this node: the refusal there is a planning error, not a
// fallback. So the engine is checked here. A file-engine table is the concrete case, since its
@@ -425,7 +323,7 @@ impl PromPlanner {
if !supports_native_raw_series(&table_meta.engine)
|| (table_meta.engine == METRIC_ENGINE_NAME && !has_verified_metric_rewrite)
{
return Ok(declined("engine_without_raw_series_scanner"));
return Ok(None);
}
// The region's raw-series preflight refuses these, and the physical layer turns that
// refusal into an error, so decline this route here instead.
@@ -441,7 +339,7 @@ impl PromPlanner {
.get(store_api::mito_engine_options::APPEND_MODE_KEY)
.is_some_and(|value| value.eq_ignore_ascii_case("true"))
{
return Ok(declined("unsupported_time_index_type_or_append_mode"));
return Ok(None);
}
// A partition column that is not a tag splits one series across regions, and a bare scan cannot
// reconcile the fragments; a raw source can, so such a selector is admitted and its input is bound per
@@ -462,10 +360,10 @@ impl PromPlanner {
.iter()
.map(|field| Self::field_column_type(input.schema(), field)),
) else {
return Ok(declined("mixed_field_kinds"));
return Ok(None);
};
if statement_kind.is_some_and(|kind| kind != field_kind) {
return Ok(declined("selectors_disagree_on_field_kind"));
return Ok(None);
}
*statement_kind = Some(field_kind);
let series_schema = Arc::new(SeriesSchema {
@@ -485,7 +383,7 @@ impl PromPlanner {
if let LogicalPlan::TableScan(table_scan) = &mut input {
table_scan.filters.append(&mut filters);
} else {
return Ok(declined("selector_plan_is_not_a_table_scan"));
return Ok(None);
}
Ok(Some((
input,
@@ -495,7 +393,6 @@ impl PromPlanner {
tags,
partitioned,
split_series,
keys_survive,
)))
}
@@ -504,48 +401,14 @@ impl PromPlanner {
prom_expr: &PromExpr,
query_engine_state: &QueryEngineState,
) -> Result<Option<LogicalPlan>> {
// With the route off nothing below runs, so reporting the answer as a fallback would
// describe a route the query never asked for.
if !self.native_fused {
return Ok(None);
}
let planned = self
match self
.try_plan_native(prom_expr, query_engine_state, FusedOutputFormat::PlainRows)
.await?;
// Whether this statement is one whose metadata drop the canonical pass collected for its
// key columns alone. The attempts below record it where they decide it; taking it here
// keeps the count one per statement and leaves nothing behind for the next one.
let skipped = SKIPPED_LABELSET_CHECK.with(|skipped| skipped.replace(None));
match planned {
.await?
{
// A whole statement answers a row table. A frame is what a composition reads from its
// operands, so the statement root is never one.
Some(NativePlanned::Rows(plan)) => {
if let Some(reason) = skipped {
crate::metrics::DUPLICATE_LABELSET_CHECK_SKIPPED_TOTAL
.with_label_values(&[reason])
.inc();
#[cfg(test)]
REPORTED_SKIPPED_LABELSET_CHECK.with(|reported| reported.set(Some(reason)));
}
Ok(Some(plan))
}
_ => {
// A declined expression is planned by the existing path and answers the same rows,
// so without this record a route that never ran would only be visible as a slower
// query. The cause is the refusal that decided the answer; a refusal from a walk
// that records nothing reports the generic cause rather than staying silent.
let reason = take_native_decline().unwrap_or(NATIVE_DECLINE_UNRECORDED);
debug!(
"Native PromQL route declined this expression ({reason}); \
the existing planner answers it"
);
crate::metrics::NATIVE_PROMQL_DECLINED_TOTAL
.with_label_values(&[reason])
.inc();
#[cfg(test)]
REPORTED_DECLINE.with(|reported| reported.set(Some(reason)));
Ok(None)
}
Some(NativePlanned::Rows(plan)) => Ok(Some(plan)),
_ => Ok(None),
}
}
@@ -560,26 +423,13 @@ impl PromPlanner {
///
/// `Box::pin` is here only because a composition and this attempt call each other, which an anonymous
/// async future cannot be sized for.
///
/// The boxed future is polled through [`GrownStack`], which is the one place this route's recursion
/// grows its stack: a composition offers each operand here, so the descent alternates between this
/// attempt and [`Self::try_plan_native_composed_binary`], and every one of those levels is charged to
/// the stack of the level above it, below the reserve `PLANNER_STACK_RED_ZONE` names.
fn try_plan_native<'a>(
&'a mut self,
prom_expr: &'a PromExpr,
query_engine_state: &'a QueryEngineState,
format: FusedOutputFormat,
) -> BoxFuture<'a, Result<Option<NativePlanned>>> {
// One attempt is one whole-expression offer: a test counts them to hold the cost of a
// refused chain to its own positions, and the report below counts what a query paid for.
#[cfg(test)]
NATIVE_ATTEMPTS.with(|attempts| attempts.set(attempts.get() + 1));
// An attempt descends the tree it was offered once per operator, and the composition below
// offers each operand in turn, so the recursion grows its stack here as well as at the
// planner's own descent: `GrownStack` grows it for the whole poll of this future, which
// covers every nested attempt the poll reaches.
Box::pin(GrownStack::new(async move {
Box::pin(async move {
let saved = self.ctx.clone();
let whole = Box::pin(self.try_plan_native_fused_with_context(
prom_expr,
@@ -598,7 +448,7 @@ impl PromPlanner {
.await;
self.ctx = saved;
composed
}))
})
}
/// Combines two native operands without a legacy join.
@@ -615,9 +465,6 @@ impl PromPlanner {
query_engine_state: &QueryEngineState,
format: FusedOutputFormat,
) -> Result<Option<NativePlanned>> {
// Neither check refuses a shape: they say this second attempt does not apply here, so they
// record no cause - the whole-expression attempt above already named the one that decided
// the answer.
if !self.native_fused
|| self.ctx.start != self.ctx.stmt_start
|| self.ctx.end != self.ctx.stmt_end
@@ -645,7 +492,7 @@ impl PromPlanner {
.await;
self.ctx = saved.clone();
let Some(NativePlanned::Frame(left)) = left? else {
return Ok(declined("left_operand_is_not_a_frame"));
return Ok(None);
};
let right = self
.try_plan_native(
@@ -656,10 +503,10 @@ impl PromPlanner {
.await;
self.ctx = saved;
let Some(NativePlanned::Frame(right)) = right? else {
return Ok(declined("right_operand_is_not_a_frame"));
return Ok(None);
};
if left.grid.spec() != right.grid.spec() {
return Ok(declined("operand_grids_differ"));
return Ok(None);
}
let NativeFramePlan {
plan: left_plan,
@@ -691,13 +538,11 @@ impl PromPlanner {
evaluated(1, &right_output),
) {
Ok(tree) => tree,
Err(DataFusionError::NotImplemented(_)) => {
return Ok(declined("no_kernel_for_the_operator_pair"));
}
Err(DataFusionError::NotImplemented(_)) => return Ok(None),
Err(error) => return Err(error).context(DataFusionPlanningSnafu),
};
let NativeExpr::Binary { matching, .. } = &tree.root().expr else {
return Ok(declined("bound_root_is_not_the_binary"));
return Ok(None);
};
let Some(root_kind) = binary_output_field_kind(
left_kind,
@@ -706,10 +551,10 @@ impl PromPlanner {
matching.return_bool,
matching.fill,
) else {
return Ok(declined("no_output_kind_for_the_pair"));
return Ok(None);
};
if !native_tree_supported(&tree) {
return Ok(declined("runtime_refuses_the_composed_tree"));
return Ok(None);
}
// The expression names the value columns, and it is the label the node reports.
let expression = expr.to_string();
@@ -720,7 +565,7 @@ impl PromPlanner {
FusedOutputFormat::PlainRows => None,
FusedOutputFormat::SeriesFrame => {
let Some(tags) = fused_output_tags(tree.root(), &source_tags) else {
return Ok(declined("composition_has_no_output_labels"));
return Ok(None);
};
Some((tree.root().output.clone(), tags))
}
@@ -735,7 +580,7 @@ impl PromPlanner {
root_kind,
)?
else {
return Ok(declined("composition_has_no_value_columns"));
return Ok(None);
};
schema
}
@@ -743,9 +588,7 @@ impl PromPlanner {
let frame = match FusedSeries::series_frame_schema(&tree) {
Ok(frame) => frame,
// A frame this composition cannot declare is a shape it does not answer.
Err(DataFusionError::Plan(_)) => {
return Ok(declined("composition_cannot_declare_a_frame"));
}
Err(DataFusionError::Plan(_)) => return Ok(None),
Err(error) => return Err(error).context(DataFusionPlanningSnafu),
};
Arc::new(
@@ -766,7 +609,7 @@ impl PromPlanner {
output_schema,
format,
) else {
return Ok(declined("fused_node_refuses_the_composition"));
return Ok(None);
};
let plan = LogicalPlan::Extension(Extension {
node: Arc::new(node),
@@ -796,7 +639,7 @@ impl PromPlanner {
format: FusedOutputFormat,
) -> Result<Option<NativePlanned>> {
if !self.native_fused {
return Ok(declined("native_route_not_requested"));
return Ok(None);
}
// The expression names the value columns of every rows layout this attempt publishes, and the
// node's own label follows the same text. It is the whole statement's expression even when a
@@ -809,23 +652,23 @@ impl PromPlanner {
if prom_expr.value_type() == ValueType::Matrix
&& (subquery_root.is_none() || self.ctx.start != self.ctx.end)
{
return Ok(declined("matrix_root_without_a_grid"));
return Ok(None);
}
let Some(stmt) = self.native_statement(prom_expr) else {
return Ok(declined("statement_has_no_evaluation_grid"));
return Ok(None);
};
// The subquery root's inner expression is compiled on the child grid, with the statement left
// untouched: `@ start()` and `@ end()` inside it resolve against the statement's bounds. A statement
// whose grid cannot be built keeps the existing plan.
let child_grid = if let Some(subquery) = subquery_root {
let Ok(outer) = statement_grid(&stmt) else {
return Ok(declined("subquery_outer_grid_unavailable"));
return Ok(None);
};
let Some(child) =
subquery_child_grid(&stmt, &outer, subquery).context(DataFusionPlanningSnafu)?
else {
return Ok(declined("subquery_has_no_child_grid"));
return Ok(None);
};
Some(child.grid)
} else {
@@ -847,12 +690,12 @@ impl PromPlanner {
&mut selectors,
&mut set_operations,
) {
return Ok(declined("selector_walk_refuses_the_statement"));
return Ok(None);
}
// The compiler decides which calls and shapes this route answers: a name it has no kernel for comes
// back as `NotImplemented`, which is a decline below.
let Ok(statement) = statement_grid(&stmt) else {
return Ok(declined("statement_grid_unavailable"));
return Ok(None);
};
// The one grid every lowering path below binds: a subquery root is compiled on its child
// grid, and the rows it publishes carry that grid, so the split, the boundary and the
@@ -863,9 +706,7 @@ impl PromPlanner {
compile_stmt.expr = effective_expr.clone();
let tree = match compile_statement_with_grid(&compile_stmt, Arc::clone(&grid)) {
Ok(tree) => tree,
Err(DataFusionError::NotImplemented(_)) => {
return Ok(declined("no_kernel_for_the_statement_shape"));
}
Err(DataFusionError::NotImplemented(_)) => return Ok(None),
Err(error) => return Err(error).context(DataFusionPlanningSnafu),
};
// A frame operand is an evaluated sample root: a composition matches series, and a
@@ -876,16 +717,16 @@ impl PromPlanner {
NativeType::Samples(schema) if schema.eval_grid.is_some()
);
if format == FusedOutputFormat::SeriesFrame && !evaluated {
return Ok(declined("frame_operand_is_not_evaluated"));
return Ok(None);
}
// Bind only input tags: this data-free tree has no input, and its tags belong to its
// output schema.
let Some(output_tags) = fused_output_tags(tree.root(), &[]) else {
return Ok(declined("data_free_root_has_no_labels"));
return Ok(None);
};
let Some(values) = native_root_fields(tree.root()) else {
return Ok(declined("data_free_root_has_no_value_columns"));
return Ok(None);
};
// A label named `time` is the one collision the default time column can have: the
// value columns are named after the expression, so they never take that name. The
@@ -976,7 +817,7 @@ impl PromPlanner {
// These matchers are alternatives the one scan below the node cannot represent, so the
// query keeps the plan the existing path builds for it.
if !selector.matchers.or_matchers.is_empty() {
return Ok(declined("or_matchers_have_no_single_raw_scan"));
return Ok(None);
}
// A selector that names no metric is answered by enumerating the schema's logical metric
// tables: one input per metric, each reading its own region, so every series keeps the
@@ -994,7 +835,7 @@ impl PromPlanner {
.preprocess_label_matchers(&ordinary_matchers, &Some(String::new()))
{
Ok(m) => m,
Err(_) => return Ok(declined("discovery_matchers_cannot_be_preprocessed")),
Err(_) => return Ok(None),
};
let effective_schema = self
.ctx
@@ -1005,7 +846,7 @@ impl PromPlanner {
// see never becomes a source, so its name reaches neither the plan nor an error.
let scope = self.table_provider.query_ctx().metric_discovery_scope();
if matches!(scope, MetricDiscoveryScope::Unavailable) {
return Ok(declined("metric_discovery_unavailable"));
return Ok(None);
}
let mut metrics = self
.table_provider
@@ -1071,7 +912,7 @@ impl PromPlanner {
// is declined for a scan this route can read, and the same refusal keeps the empty
// input from claiming a field layout the matcher would have decided.
if self.ctx.field_column_matcher.is_some() {
return Ok(declined("field_selector_on_an_empty_input"));
return Ok(None);
}
prepared.push(empty_discovered_input(
self.ctx
@@ -1089,14 +930,10 @@ impl PromPlanner {
// makes the whole input partitioned.
let mut discovered_partitioned = false;
let mut discovered_split_series = false;
// Every discovered metric's own key columns decide its series, so the union
// survives the drop only when all of them do - and a metric whose keys do not
// is enough to collect the whole input.
let mut discovered_keys_survive = true;
for table in metrics {
self.ctx.table_name = Some(table.table_info().name.clone());
if self.setup_context().await?.is_some() {
return Ok(declined("discovered_table_has_no_data"));
return Ok(None);
}
let Some((
scan,
@@ -1106,7 +943,6 @@ impl PromPlanner {
mut tags,
partitioned,
split_series,
keys_survive,
)) = self
.prepare_one_scan(
matchers.clone(),
@@ -1117,17 +953,14 @@ impl PromPlanner {
)
.await?
else {
// A refusal here is the selector's own: `prepare_one_scan` recorded
// its cause, and naming this pass again would overwrite it.
return Ok(None);
};
discovered_partitioned |= partitioned;
discovered_split_series |= split_series;
discovered_keys_survive &= keys_survive;
// The union reads every discovered metric, so it is region-local only when
// every metric it reads is.
let Some(metric) = metric else {
return Ok(declined("discovered_selector_names_no_metric"));
return Ok(None);
};
// The expression names no metric, so each source reports its own: the tag is
// what tells two metrics with equal ordinary labels apart, and it is what a
@@ -1180,7 +1013,7 @@ impl PromPlanner {
}
}
let Some((schema, time, mut tags, single_metric)) = selector_metadata else {
return Ok(declined("discovered_selectors_disagree_on_their_layout"));
return Ok(None);
};
tags.sort();
tags.dedup();
@@ -1205,17 +1038,16 @@ impl PromPlanner {
tags,
discovered_partitioned,
discovered_split_series,
discovered_keys_survive,
));
continue;
}
let matchers =
match self.preprocess_label_matchers(&selector.matchers, &selector.name) {
Ok(m) => m,
Err(_) => return Ok(declined("selector_matchers_cannot_be_preprocessed")),
Err(_) => return Ok(None),
};
if self.setup_context().await?.is_some() {
return Ok(declined("selector_has_no_data"));
return Ok(None);
}
let Some(one) = self
.prepare_one_scan(
@@ -1227,7 +1059,6 @@ impl PromPlanner {
)
.await?
else {
// `prepare_one_scan` recorded why, above.
return Ok(None);
};
prepared.push(one);
@@ -1237,7 +1068,6 @@ impl PromPlanner {
.await;
self.ctx = original_context;
let Some((mut prepared, logical_ids_differ, statement_kind)) = preparation? else {
// The selector that refused recorded its own cause; this pass only carries it up.
return Ok(None);
};
// `absent_over_time` reads its window through a fold that keeps the metric name, so the segment that
@@ -1252,7 +1082,7 @@ impl PromPlanner {
.iter()
.any(|(scan, _, metric, ..)| !is_empty_discovered_input(scan) && metric.is_none())
{
return Ok(declined("presence_over_a_nameless_selector"));
return Ok(None);
}
// Selectors over one table share one region, so the node runs there over every scan. When they read
// more than one table, each scan answers from its own region and the node runs above them; see
@@ -1297,12 +1127,12 @@ impl PromPlanner {
prepared[index].1 = Arc::clone(&schema);
}
// A real input names the output's time index and, on one table, binds every scan.
let Some((_, _, metric, time, tags, _, _, _)) = prepared
let Some((_, _, metric, time, tags, _, _)) = prepared
.iter()
.find(|(scan, ..)| !is_empty_discovered_input(scan))
.or_else(|| prepared.first())
else {
return Ok(declined("statement_prepared_no_readable_input"));
return Ok(None);
};
let (metric, time, tags) = (metric.clone(), time.clone(), tags.clone());
// The compiler binds one input per selector in the same pre-order `collect_selectors`
@@ -1367,9 +1197,7 @@ impl PromPlanner {
) {
Ok(v) => v,
// The compiler refusing a shape is this route declining it.
Err(DataFusionError::NotImplemented(_)) => {
return Ok(declined("no_kernel_for_the_statement_shape"));
}
Err(DataFusionError::NotImplemented(_)) => return Ok(None),
Err(e) => return Err(e).context(DataFusionPlanningSnafu),
};
if compiled.inputs.len() != prepared.len() {
@@ -1379,11 +1207,11 @@ impl PromPlanner {
))
.context(DataFusionPlanningSnafu);
}
for (index, (compiled_input, (scan, _, _, time, _, _, _, _))) in
for (index, (compiled_input, (scan, _, _, time, _, _, _))) in
compiled.inputs.iter().zip(prepared.iter_mut()).enumerate()
{
if compiled_input.source_index != index {
return Ok(declined("compiled_sources_out_of_order"));
return Ok(None);
}
// The range this selector reads is the compiler's own: a window and an instant differ in which
// side of the lookback or range their scan starts on, and an anchor can move the whole range
@@ -1481,7 +1309,7 @@ impl PromPlanner {
}
let source_tags = prepared
.iter()
.map(|(_, _, _, _, tags, _, _, _)| tags.clone())
.map(|(_, _, _, _, tags, _, _)| tags.clone())
.collect::<Vec<_>>();
// The complete runtime label set of each input: a scan bound to one metric names every series it
// hands over, so `__name__` is one of its labels even though it is not a tag column. Rewriting the
@@ -1489,7 +1317,7 @@ impl PromPlanner {
// declares the name among its tags itself.
let runtime_source_tags = prepared
.iter()
.map(|(_, _, metric, _, tags, _, _, _)| {
.map(|(_, _, metric, _, tags, _, _)| {
let mut tags = tags.clone();
if metric.is_some() && !tags.iter().any(|tag| tag == METRIC_NAME) {
tags.push(METRIC_NAME.to_string());
@@ -1497,20 +1325,6 @@ impl PromPlanner {
tags
})
.collect::<Vec<_>>();
// The other half of what the canonical pass judges a dropped label by: the one metric each
// input is bound to. A scan of one table names every series it hands over, so `__name__` is
// the same for all of them and dropping it tells none of them apart; a discovered selector
// that expands into several metrics binds no single name, which is the fact that keeps
// `rate` over it collected.
let runtime_source_metrics = prepared
.iter()
.map(|(_, _, metric, _, _, _, _, _)| metric.clone())
.collect::<Vec<_>>();
// The drop's other half, and the one no tree can answer about itself: whether every input
// keys its series by columns that always decode a value. One input that cannot say so - a
// nullable primary-key column, or a discovered selector whose metrics disagree - is enough
// for the pass to collect the drop rather than trust it.
let keys_survive = prepared.iter().all(|input| input.7);
// Whether any input's table is split into regions along a partition column. Only the canonical
// boundary claims a whole-query label-set answer, and a region-local producer cannot see the pieces
// another region holds.
@@ -1522,26 +1336,10 @@ impl PromPlanner {
let tree = match promql::native::canonicalize::merge_label_rewrites(
&compiled.tree,
&runtime_source_tags,
&runtime_source_metrics,
keys_survive,
) {
Ok(tree) => tree,
Err(_) => return Ok(declined("label_rewrites_do_not_merge")),
Err(_) => return Ok(None),
};
// What those collections cost: a node that drops the metadata labels is proved by its labels
// and its scan's key columns together now, so a drop the labels alone would have proved is
// collected exactly when the keys do not survive it - the pair a primary-key column can
// spell as `NULL` in one series and `''` in another. Counting the statements collected for
// that reason is what keeps the price of the fact observable.
if promql::native::plan::leaves_a_drop_to_the_storage_keys(
&tree,
&runtime_source_tags,
&runtime_source_metrics,
keys_survive,
) {
SKIPPED_LABELSET_CHECK
.with(|skipped| skipped.set(Some(SKIPPED_LABELSET_CHECK_NULLABLE_KEY)));
}
let root = tree.root();
// The placement facts the split and the segmentation below are chosen from, and the same
// facts that say whether one node can run over the scans at all.
@@ -1581,13 +1379,13 @@ impl PromPlanner {
.collect::<Vec<_>>();
let per_source = prepared
.iter()
.map(|(_, _, metric, time_index, tags, _, _, _)| {
.map(|(_, _, metric, time_index, tags, _, _)| {
(metric.clone(), time_index.clone(), tags.clone())
})
.collect::<Vec<_>>();
let children = prepared
.into_iter()
.map(|(scan, schema, metric, _, tags, _, split_series, _)| {
.map(|(scan, schema, metric, _, tags, _, split_series)| {
// A discovered selector already carries its own sources: a single metric is one
// `RawSeriesSource`, several are a `Union` of them, and each names the metric it
// reads. Wrapping either again would nest a source in a source.
@@ -1643,35 +1441,38 @@ impl PromPlanner {
// A tree with no canonical merge claims no label-set answer, so it is untouched: a
// per-series query on a partitioned table keeps fusing.
if !spans_regions && any_partitioned && contains_canonical_merge(root) {
return Ok(declined("canonical_merge_over_partitioned_sources"));
return Ok(None);
}
// The kind the tree's root answers with, which every output column follows. It is not the kind the
// inputs carry: `count_over_time`, `histogram_count` and `timestamp` reduce a histogram to a
// float, while a selector, a histogram-answering range function and a label rewrite pass it on.
// A canonical merge hands on the samples it was given, so an answer kind crosses it unchanged:
// a histogram answer keeps its frames over the boundary, and the router merges them with the
// same kernel it merges floats with.
// A shape whose kind cannot be named keeps the existing plan rather than declaring a column the
// first batch might not fill.
let root_kind = match statement_kind {
Some(FieldKind::Histogram) => {
// A histogram answer standing on a canonical merge would run through the canonical
// boundary carrying histogram frames, a pairing nothing exercises yet, so such a
// statement keeps the existing plan. Every merge counts, not only a root one.
if merge_answers_histogram(root, FieldKind::Histogram) {
return Ok(None);
}
match root.output_field_kind(Some(FieldKind::Histogram)) {
Some(root_kind) => root_kind,
None => return Ok(declined("root_declares_no_field_kind")),
None => return Ok(None),
}
}
Some(FieldKind::Float) | None => FieldKind::Float,
};
let frame_metadata = if format == FusedOutputFormat::SeriesFrame {
let NativeType::Samples(schema) = &root.output else {
return Ok(declined("frame_operand_is_not_samples"));
return Ok(None);
};
if schema.eval_grid.is_none() {
return Ok(declined("frame_operand_has_no_grid"));
return Ok(None);
}
// A segment publishes frames, so it reports no name column of its own.
let Some(output_tags) = fused_output_tags(root, &source_tags) else {
return Ok(declined("root_has_no_output_labels"));
return Ok(None);
};
Some((root.output.clone(), output_tags))
} else {
@@ -1685,20 +1486,20 @@ impl PromPlanner {
if let Some(kind @ FieldKind::Histogram) = statement_kind
&& ((multi_input && per_source_inputs) || aggregate_reads_histogram(root, kind))
{
return Ok(declined("histogram_aggregate_needs_state_this_route_lacks"));
return Ok(None);
}
// The split keeps its prefix per-series, which is why it declines an operation whose
// answer depends on the order its input arrives in. A node over more than one input reads
// its inputs' partitions in the order they became ready, so that refusal holds there too.
if multi_input && depends_on_input_order(root) {
return Ok(declined("answer_depends_on_input_order"));
return Ok(None);
}
// The native output schema can name only compiled value fields, the time index and the tag
// columns a source declares. A discovered source declares `__name__` because the name tells two
// metrics with equal labels apart (#10106); a named source does not, so a tree reading more than
// one such source cannot report the distinct groups and declines.
if multi_input && groups_by_metric_name(root) {
return Ok(declined("grouping_by_metric_name_over_several_sources"));
return Ok(None);
}
// Catch the shape refusals before choosing native execution: errors raised during physical
// planning cannot fall back. This includes unsupported aggregates and `topk`, `limitk` or
@@ -1706,7 +1507,7 @@ impl PromPlanner {
// tree. A plan split over regions commits to the fused node without the split or a segment
// above it, so it checks itself the same way before that choice.
if (multi_input || per_source_inputs) && !native_tree_supported(&tree) {
return Ok(declined("runtime_refuses_the_tree"));
return Ok(None);
}
// A tree that ends in a canonical merge answers one series per label set, and the pieces of one
// set were spread over the scan's partitions by the rewrite that created them. Lowering that
@@ -1759,7 +1560,7 @@ impl PromPlanner {
if root_kind != FieldKind::Float
&& !(root_kind == FieldKind::Histogram && histogram_state)
{
return Ok(declined("split_aggregate_state_cannot_carry_the_answer"));
return Ok(None);
}
// The row-value layout is what the split is validated against whichever format the
// caller asked for: it names one column per value the partial state accumulates,
@@ -1767,16 +1568,16 @@ impl PromPlanner {
let Some(value_schema) =
Self::fused_root_schema(&expression, &tree, &source_tags, &time, root_kind)?
else {
return Ok(declined("root_has_no_value_columns"));
return Ok(None);
};
// A frame operand also needs the evaluated layout its consumers decode, which is
// the metadata derived above: a request that could not name one keeps the existing
// plan rather than publishing rows to a frame consumer.
if format == FusedOutputFormat::SeriesFrame && frame_metadata.is_none() {
return Ok(declined("frame_operand_needs_an_evaluated_layout"));
return Ok(None);
}
let Some(rawinput) = children.into_iter().next() else {
return Ok(declined("split_aggregate_has_no_input"));
return Ok(None);
};
let publish_grid = Arc::clone(&grid);
let node = plan_split_aggregate(
@@ -1858,7 +1659,7 @@ impl PromPlanner {
// contribution the consumer needs. Keep that consumer on the existing plan; a tree over more
// than one input is not this shape.
if required_placement && !multi_input {
return Ok(declined("cross_series_consumer_over_one_scan"));
return Ok(None);
}
}
// The tags are asked for before the fields, because the walk answers for a tree the
@@ -1869,14 +1670,12 @@ impl PromPlanner {
NativeType::Scalar(_) => Vec::new(),
NativeType::Samples(_) => match fused_output_tags(tree.root(), &source_tags) {
Some(output_tags) => with_row_name_column(tree.root(), output_tags),
None => return Ok(declined("root_has_no_output_labels")),
None => return Ok(None),
},
NativeType::AggregateState(_) => {
return Ok(declined("aggregate_state_is_not_a_statement_answer"));
}
NativeType::AggregateState(_) => return Ok(None),
};
let Some(root_fields) = native_root_fields(tree.root()) else {
return Ok(declined("root_has_no_value_columns"));
return Ok(None);
};
let output_schema = match format {
FusedOutputFormat::PlainRows => {
@@ -2015,18 +1814,8 @@ impl PromPlanner {
fn empty_discovered_input(time_index: String) -> Result<PreparedInput> {
let schema = empty_discovered_series_schema();
let input = empty_discovered_frame(&schema)?;
// No table stands behind this input, so no region splits it along a partition column - and no
// metadata states what its keys would survive, which is the answer that collects.
Ok((
input,
schema,
None,
time_index,
Vec::new(),
false,
false,
false,
))
// No table stands behind this input, so no region splits it along a partition column.
Ok((input, schema, None, time_index, Vec::new(), false, false))
}
fn empty_discovered_frame(schema: &SeriesSchema) -> Result<LogicalPlan> {
@@ -3446,9 +3235,8 @@ mod tests {
fused_output_tags(tree.root(), &columns),
"a written name is a column the output publishes"
);
let collected =
promql::native::canonicalize::merge_label_rewrites(&tree, &runtime, &[], true)
.expect("the pass produces a legal tree");
let collected = promql::native::canonicalize::merge_label_rewrites(&tree, &runtime)
.expect("the pass produces a legal tree");
assert!(
matches!(collected.root().expr, NativeExpr::Merge { .. }),
"a name rewrite writes an existing runtime label: {collected}"
@@ -3495,17 +3283,16 @@ mod tests {
]];
let tree =
compiled_test_tree(r#"label_replace(abs(x), "__name__", "same", "missing", "(.*)")"#);
let collected =
promql::native::canonicalize::merge_label_rewrites(&tree, &runtime, &[], true)
.expect("the pass produces a legal tree");
let collected = promql::native::canonicalize::merge_label_rewrites(&tree, &runtime)
.expect("the pass produces a legal tree");
assert!(
matches!(collected.root().expr, NativeExpr::Merge { .. }),
"the merge sits above the name rewrite: {collected}"
);
assert_eq!(
2,
1,
collected.to_string().matches("Merge").count(),
"the name rewrite and the name drop below it are each collected: {collected}"
"exactly one merge: {collected}"
);
// The rewrite writes a name over a dropped one, so the output publishes a name column.
assert_eq!(
@@ -3516,14 +3303,11 @@ mod tests {
// A fresh ordinary destination above the same kernel is still uncollected: no scan can carry
// an ordinary label it does not declare.
let fresh = compiled_test_tree(r#"label_replace(abs(x), "dst", "$1", "host", "(.*)")"#);
let collected =
promql::native::canonicalize::merge_label_rewrites(&fresh, &runtime, &[], true)
.expect("the pass produces a legal tree");
assert_eq!(
1,
collected.to_string().matches("Merge").count(),
"the fresh ordinary destination adds no collection of its own, and `abs` keeps the one \
its name drop needs: {collected}"
let collected = promql::native::canonicalize::merge_label_rewrites(&fresh, &runtime)
.expect("the pass produces a legal tree");
assert!(
!collected.to_string().contains("Merge"),
"a fresh ordinary destination does not alias: {collected}"
);
}
File diff suppressed because it is too large Load Diff
@@ -412,7 +412,6 @@ async fn delta_mixed_ranges_drop_and_float_ranges_sum() {
ctx: PromPlannerContext::default(),
promql_annotations: Some(collector.clone()),
native_fused: false,
native_binary_offer: true,
};
let left_context = direct_or_context("lhs", &["job", marker], "v");
let right_context = direct_or_context("rhs", &["job", marker], "v");
@@ -715,7 +714,6 @@ async fn binary_joins_align_only_the_temporality_marker() {
ctx: PromPlannerContext::default(),
promql_annotations: None,
native_fused: false,
native_binary_offer: true,
};
let joined = planner
.join_on_non_field_columns(
@@ -756,7 +754,6 @@ async fn binary_joins_align_only_the_temporality_marker() {
ctx: PromPlannerContext::default(),
promql_annotations: None,
native_fused: false,
native_binary_offer: true,
};
let set = planner
.set_op_on_non_field_columns(
@@ -813,7 +810,6 @@ async fn binary_joins_align_only_the_temporality_marker() {
ctx: PromPlannerContext::default(),
promql_annotations: None,
native_fused: false,
native_binary_offer: true,
};
let set = planner
.set_op_on_non_field_columns(
@@ -927,7 +923,6 @@ async fn binary_joins_align_dictionary_temporality_marker_with_tagless_vector()
ctx: PromPlannerContext::default(),
promql_annotations: None,
native_fused: false,
native_binary_offer: true,
};
let scan = |name, table: Arc<MemTable>| {
LogicalPlanBuilder::scan(name, provider_as_source(table), None)
-21
View File
@@ -340,27 +340,6 @@ fn native_anchored_timestamp_selection_boundaries() {
fn timestamp_refuses_two_series_that_collapse_to_one_label_set() {
let stmt = stmt("timestamp(a @ 1)", 0, 4000, 3000);
let tree = compile(&stmt, schema(1), SeriesOrdering::KeyThenTime).unwrap();
// The refusal is the canonical pass's, so this test makes the pass the planner makes: the facts
// it hands over for a scan bound to one metric are the labels that scan's series carry -
// `__name__` among them, named by the binding - and the one metric it is bound to. `timestamp`
// drops the metadata labels, and `__type__` is one of them, so two of that scan's series can
// become one label set; the pass therefore collects the pieces above the node, and the merge is
// what reports the collision. The node itself keeps no table to check.
let source_tags = vec![vec![
"job".to_string(),
"__type__".to_string(),
"__name__".to_string(),
]];
let source_metrics = vec![Some(Arc::<str>::from("a"))];
// The scan's key columns always decode a value here, so the labels alone decide: `__type__` in
// the tags is what keeps this drop collected.
let tree = promql::native::canonicalize::merge_label_rewrites(
&tree,
&source_tags,
&source_metrics,
true,
)
.expect("the canonical pass accepts the tree");
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 24));
let state = NativeExecutionState::try_new(&tree, pool, None).unwrap();
let mut executor = NativeExecutor::new(state);
+47 -319
View File
@@ -209,7 +209,6 @@ macro_rules! http_tests {
test_remote_write_v2_histogram_flush_native_consistency,
test_remote_write_v2_histogram_native_partitions,
test_remote_write_v2_histogram_native_range_function,
test_remote_write_v2_histogram_native_nullable_tag_boundary,
test_prometheus_remote_write_batched,
test_prometheus_remote_write_batched_mixed_time_index_units,
test_prometheus_remote_write_batched_interceptor_time_index_units,
@@ -1702,13 +1701,10 @@ fn assert_native_plan(
) -> Value {
let fused = {
let mut nodes = plan_nodes_named(plan, "FusedSeriesExec");
// One node, or the pair a collected metadata drop is: the producer that runs the tree on
// each partition and the consumer that converts the assembled frames into rows. These
// fixture tables declare their tag column nullable, so the drop is collected and the pair is
// what a vector statement plans.
assert!(
(1..=2).contains(&nodes.len()),
"{context}: expected one fused node or the boundary's pair: {nodes:?}"
assert_eq!(
nodes.len(),
1,
"{context}: expected one fused node: {nodes:?}"
);
nodes.remove(0)
};
@@ -2551,25 +2547,6 @@ pub async fn test_prometheus_native_promql_route(store_type: StorageType) {
instance.guard.remove_all().await;
}
/// The fused node `node` is, as its own line renders it.
fn describe_node(node: &Arc<dyn ExecutionPlan>) -> String {
datafusion::physical_plan::displayable(node.as_ref())
.one_line()
.to_string()
}
/// The fused nodes of one request, as their own node lines render them, for a failure message.
fn describe_nodes(nodes: &[Arc<dyn ExecutionPlan>]) -> Vec<String> {
nodes
.iter()
.map(|node| {
datafusion::physical_plan::displayable(node.as_ref())
.one_line()
.to_string()
})
.collect()
}
/// Keeps the native fused node and the raw scan the request executed, for the figures only they
/// record.
///
@@ -2596,42 +2573,12 @@ impl NativeFusedObserver {
assert_eq!(
captured.len(),
1,
"`{context}`: the request executed {} fused nodes: {:?}",
captured.len(),
describe_nodes(&captured)
"`{context}`: the request executed {} fused nodes: {captured:?}",
captured.len()
);
captured.remove(0)
}
/// The fused node that ran the statement's own tree.
///
/// A statement whose metadata drop is collected runs a canonical boundary: the producer runs the
/// tree on each scan partition and the consumer above the router only converts the assembled
/// frames into the statement's rows. Every figure below is read off the producer, because it is
/// the node that binds the source, reads the scan and reports the partitions it ran on; a
/// statement with nothing collected runs that node alone.
fn take_statement(&self, context: &str) -> Arc<dyn ExecutionPlan> {
let mut captured = std::mem::take(&mut *self.0.lock().unwrap());
assert!(
!captured.is_empty(),
"`{context}`: the request executed no fused node, so it left the native route"
);
// At most the pair a boundary is: one producer and the consumer above it.
assert!(
captured.len() <= 2,
"`{context}`: the request executed {} fused nodes: {:?}",
captured.len(),
describe_nodes(&captured)
);
if let Some(index) = captured
.iter()
.position(|node| promql::extension_plan::is_canonical_producer_exec(node.as_ref()))
{
return captured.remove(index);
}
captured.remove(0)
}
/// The one raw scan the request just executed.
///
/// The fused node reads its region through a router over a merge scan, whose region half is
@@ -2825,7 +2772,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
// The body is drained, so every partition has finished and the node this statement ran is
// complete; its own registrations - not the flat set the JSON view reads - carry the
// per-partition figures.
let exec = observer.take_statement(query);
let exec = observer.take(query);
assert_native_plan(&plan, &exec, scan_partitions, query);
assert_native_partition_output(&exec, scan_partitions, expected.len(), query);
@@ -2884,12 +2831,10 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
"`{query}` did not execute the fused route"
);
let fused_nodes = plan_nodes_named(&plan, "FusedSeriesExec");
// The drop `* 100` makes over these fixture tables' nullable tag column is collected, so the
// native plan is that boundary: the producer that runs the tree and the consumer above it. An
// empty match changes which series are read, not which stages the statement is lowered into.
assert!(
(1..=2).contains(&fused_nodes.len()),
"`{query}`: expected one fused node or the boundary's pair: {fused_nodes:?}"
assert_eq!(
fused_nodes.len(),
1,
"`{query}`: expected one fused node: {fused_nodes:?}"
);
let scans = plan_nodes_named(&plan, "SeriesScanExec");
assert_eq!(
@@ -2900,7 +2845,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
// The body is drained, so the one fused node the request ran has finished. Its own child list is
// what says how many plans it reads, and its own line is where the tree it ran is still one text:
// the JSON view lists that tree's nodes as children of the operator, next to the plans it reads.
let exec = observer.take_statement(query);
let exec = observer.take(query);
assert_eq!(
exec.children().len(),
1,
@@ -5872,203 +5817,9 @@ pub async fn test_remote_write_v2_histogram_native_range_function(store_type: St
body["data"]["result"][0]["histogram"][1]["sum"], "30",
"`sum_over_time` adds the sums as {PROMETHEUS_REFERENCE} does"
);
// `sum_over_time` over a native histogram drops the metadata labels, and this table's tags are
// nullable, so the canonical pass collects that drop: the answer stands on a canonical merge
// carrying histogram frames. Lowering such a boundary is two nodes, not one - a producer that
// runs the statement's own tree per partition and exports a frame, and the consumer above the
// router that converts the merged frames into the statement's rows. Both are the fused route;
// the answer asserted above is what proves the histograms survived the crossing.
let fused = observer.take_all("histogram sum_over_time");
assert_eq!(
2,
fused.len(),
"the canonical boundary is a producer and a consumer: {fused:?}"
);
assert!(
fused.iter().all(|node| node.name() == "FusedSeriesExec"),
"the boundary executed a node that is not the fused producer: {fused:?}"
);
}
/// A native histogram answer crosses a canonical merge on a real metric table.
///
/// Every tag column of a metric-engine table is nullable, so `keys_survive_metadata_drop` is false
/// on one and every statement whose kernel drops the metadata labels pays a canonical merge - the
/// `NULL`/`''` pair a drop normalizes into one label set is what the collection is for. For a
/// histogram answer that merge carries histogram frames, which is the pairing this test pins end to
/// end: the producer routes the pieces of one label set, the owner merges them with the same kernel
/// it merges floats with, and the consumer answers what it collected.
///
/// The statement shapes here are the two sides of that crossing: `sum_over_time(h[1m])` answers the
/// histograms the merge collected, and `histogram_count(sum_over_time(h[1m]))` reduces them to
/// floats above it - so the boundary carries histograms while the statement answers a float.
pub async fn test_remote_write_v2_histogram_native_nullable_tag_boundary(store_type: StorageType) {
let observer = Arc::new(NativeFusedObserver::default());
let plugins = Plugins::default();
let wrapper: PhysicalPlanWrapperRef = observer.clone();
plugins.insert(wrapper);
let instance = tests_integration::standalone::GreptimeDbStandaloneBuilder::new(
"remote_write_v2_histogram_native_nullable_tag_boundary",
)
.with_default_store_type(store_type)
.with_query_options(QueryOptions {
experimental_native_promql: true,
..Default::default()
})
.with_plugin(plugins)
.build()
.await;
let server = build_test_prom_server(instance.fe_instance().clone(), false, None)
.with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap())
.build();
let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
write_native_histogram(
&client,
&[
(prom_store::METRIC_NAME_LABEL, "native_histogram_nullable"),
("job", "api"),
],
1000,
5,
10.0,
)
.await;
write_native_histogram(
&client,
&[
(prom_store::METRIC_NAME_LABEL, "native_histogram_nullable"),
("job", "api"),
],
3000,
8,
20.0,
)
.await;
// The premise of this test, read off the table the writes created: the `job` tag is nullable, so
// a drop of the metadata labels is collected for the key columns rather than proved by them.
validate_data(
"native histogram nullable tag",
&client,
"select column_name, is_nullable from information_schema.columns \
where table_schema = 'public' and table_name = 'native_histogram_nullable' \
and column_name = 'job'",
"[[\"job\",\"YES\"]]",
)
.await;
// A histogram answer standing on the collected merge: `sum_over_time` over a native histogram
// drops the metadata labels, so the two writes' histograms travel through the boundary and the
// answer is the one histogram the merge collected.
observer.clear();
let response = client
.get("/v1/prometheus/api/v1/query?query=sum_over_time(native_histogram_nullable%5B1m%5D)&time=3")
.header("x-greptime-hints", "query.experimental_native_promql=true")
.send()
.await;
assert_eq!(response.status(), StatusCode::OK);
let body: Value = response.json().await;
assert_eq!(body["data"]["resultType"], "vector", "{body}");
assert_eq!(body["data"]["result"][0]["histogram"][0], 3.0, "{body}");
assert_eq!(
body["data"]["result"][0]["histogram"][1]["count"], "13",
"`sum_over_time` adds the counts of the merged histograms: {body}"
);
assert_eq!(
body["data"]["result"][0]["histogram"][1]["sum"], "30",
"`sum_over_time` adds the sums of the merged histograms: {body}"
);
let fused = observer.take_all("histogram answer over a nullable tag");
assert_eq!(
2,
fused.len(),
"the collected drop is a producer and a consumer: {fused:?}"
);
assert!(
fused.iter().all(|node| node.name() == "FusedSeriesExec"),
"the histogram boundary left the native route: {fused:?}"
);
// The same boundary with a float answer above it: `histogram_count` maps the histograms the
// merge collected to floats, so the frames crossing the boundary are still histograms while the
// statement publishes a float column. This is the shape the removed fallback used to refuse for
// the histogram case and the only one that ever worked here, and both now take one route.
observer.clear();
let response = client
.get("/v1/prometheus/api/v1/query?query=histogram_count(sum_over_time(native_histogram_nullable%5B1m%5D))&time=3")
.header("x-greptime-hints", "query.experimental_native_promql=true")
.send()
.await;
assert_eq!(response.status(), StatusCode::OK);
let body: Value = response.json().await;
assert_eq!(body["data"]["resultType"], "vector", "{body}");
assert_eq!(
body["data"]["result"][0]["value"][1].as_str(),
Some("13.0"),
"`histogram_count` counts the merged histograms' samples: {body}"
);
let fused = observer.take_all("float answer over a histogram boundary");
assert_eq!(
2,
fused.len(),
"the collected drop is a producer and a consumer: {fused:?}"
);
assert!(
fused.iter().all(|node| node.name() == "FusedSeriesExec"),
"the float answer left the native route: {fused:?}"
);
// `rate` is the same crossing over a window: a histogram answer whose count and sum the merge
// carries, compared against the existing plan's answer so the crossing is judged by the number
// rather than by the shape alone.
let url = "/v1/prometheus/api/v1/query?query=rate(native_histogram_nullable%5B5m%5D)&time=4";
observer.clear();
let native = client
.get(url)
.header("x-greptime-hints", "query.experimental_native_promql=true")
.send()
.await;
assert_eq!(native.status(), StatusCode::OK);
let native: Value = native.json().await;
let legacy = client.get(url).send().await;
assert_eq!(legacy.status(), StatusCode::OK);
let legacy: Value = legacy.json().await;
assert_eq!(
native["data"]["result"], legacy["data"]["result"],
"`rate(h[5m])`: the native answer differs from the existing plan's"
);
// The two writes 2s apart are `count` 5 and `8` with `sum` 10 and 20, and the 5m window at the
// instant `4` extrapolates that over the range: (8 - 5) / 300s and (20 - 10) / 300s, in seconds.
assert_eq!(
(
native["data"]["result"][0]["histogram"][1]["count"].as_str(),
native["data"]["result"][0]["histogram"][1]["sum"].as_str(),
),
(Some("0.02"), Some("0.06666666666666667")),
"`rate(h[5m])` answered {native}"
);
let fused = observer.take_all("histogram rate over a nullable tag");
let descriptions = fused
.iter()
.map(|node| {
datafusion::physical_plan::displayable(node.as_ref())
.one_line()
.to_string()
})
.collect::<Vec<_>>();
assert!(
descriptions
.iter()
.all(|description| description.starts_with("FusedSeriesExec")),
"`rate(h[5m])` left the native route: {descriptions:?}"
);
// How many fused nodes a raw window's boundary executes is a property of where its regions run
// it, so this asks what the executed plan carries instead: a canonical merge standing on the
// histogram window, which is the pairing the answer above is computed through.
assert!(
descriptions.iter().any(|description| description
.contains("Merge: canonical label-set merge")
&& description.contains("Window: rate ")),
"`rate(h[5m])` executed no canonical merge over its histogram window: {descriptions:?}"
observer.take("histogram sum_over_time").name(),
"FusedSeriesExec"
);
}
@@ -6146,20 +5897,10 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag
(expected_count.to_string(), expected_sum.to_string()),
"{query}: {body}"
);
// Scalar arithmetic over a histogram drops the metadata labels, and this table's tags are
// nullable, so the answer stands on a canonical merge that carries histogram frames. The
// boundary is two fused nodes - the producer that scales each partition's histograms and the
// consumer above the router that answers the statement - and the scaled counts and sums
// asserted above are what proves the payload crossed it intact.
let fused = observer.take_all(query);
assert_eq!(
2,
fused.len(),
"{query}: the canonical boundary is a producer and a consumer: {fused:?}"
);
assert!(
fused.iter().all(|node| node.name() == "FusedSeriesExec"),
"{query} left the native route: {fused:?}"
observer.take("histogram arithmetic").name(),
"FusedSeriesExec",
"{query} left the native route"
);
}
@@ -6178,18 +5919,12 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag
Some(0),
"`2 / h` should drop the histogram position: {body}"
);
// `2 / h` drops the histogram position and answers nothing, but it is still the same statement
// kind on the same collected drop, so it runs through the same two-node boundary: the producer
// scales each partition's histograms, drops them, and the consumer answers the empty result.
let fused = observer.take_all("histogram arithmetic: scalar over histogram");
assert_eq!(
2,
fused.len(),
"`2 / h`: the canonical boundary is a producer and a consumer: {fused:?}"
);
assert!(
fused.iter().all(|node| node.name() == "FusedSeriesExec"),
"`2 / h` left the native route: {fused:?}"
observer
.take("histogram arithmetic: scalar over histogram")
.name(),
"FusedSeriesExec",
"`2 / h` left the native route"
);
}
@@ -7548,25 +7283,16 @@ pub async fn test_remote_write_v2_histogram_sum_and_avg_native_route(store_type:
&format!("{query}: the existing path answered something else: {legacy}"),
);
// Both halves of the split executed: a partial producer that folds its own input and a final
// consumer that merges the partials above the state router. The capture holds them once per
// stage that ran the split, so the count is not pinned; what is asserted is that every node
// is a fused stage of that split and that both halves are there - one alone would mean the
// other stage fell back to the existing plan.
// Both halves of the split executed: the partial producer folded each partition and the
// final consumer merged that state into the answer above.
let fused = observer.take_all(query);
let described = describe_nodes(&fused);
assert!(
described
.iter()
.all(|node| node.contains(" partial") || node.contains(" final")),
"{query}: a node of the capture is not one half of the split: {described:?}"
assert_eq!(
fused.len(),
2,
"{query}: expected the split's two nodes: {fused:?}"
);
for half in [" partial", " final"] {
assert!(
described.iter().any(|node| node.contains(half)),
"{query}: the split ran without its{half} half: {described:?}"
);
}
let consumer = answer_consumer(&fused);
assert_eq!(consumer.name(), "FusedSeriesExec");
}
}
@@ -7647,26 +7373,28 @@ pub async fn test_remote_write_v2_histogram_avg_native_partitions(store_type: St
"the partitioned average is wrong: {partitioned}"
);
let fused = observer.take_all("avg over partitions");
let consumer = answer_consumer(&fused);
assert_eq!(consumer.name(), "FusedSeriesExec");
assert_eq!(
fused.len(),
2,
"the split's partial and final nodes both execute: {fused:?}"
);
// The producer runs once per partition that read rows, which is what makes this a merge and not
// a single fold, so the figures this test reads are the partial stage's: it is the node whose
// tree carries the fold before the state router, and it is the only node that reports them.
// a single fold.
let producer = fused
.iter()
.find(|node| describe_node(node).contains(" partial"))
.expect("the split's partial producer");
// The per-partition figures are the ones the node registered per partition, which is the view
// `node_metric_set` reports; the node's own aggregated set sums them into one entry.
let metrics = node_metric_set(producer);
.find(|node| !Arc::ptr_eq(node, consumer))
.expect("the partial producer");
let metrics = producer.metrics().expect("producer metrics");
let mut partitions: BTreeMap<usize, usize> = BTreeMap::new();
for metric in metrics.iter() {
let Some(partition) = metric.partition() else {
continue;
};
let MetricValue::Count { count, .. } = metric.value() else {
continue;
};
if metric.value().name() == "native.root.output_values" {
*partitions.entry(partition).or_insert(0usize) += count.value();
if metric.value().name() == "native.root.output_values"
&& let MetricValue::Count { count, .. } = metric.value()
{
*partitions
.entry(metric.partition().expect("partition label"))
.or_insert(0usize) += count.value();
}
}
assert_eq!(