diff --git a/Cargo.lock b/Cargo.lock index fd31d88c4dc..6c3fef5dc14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12087,6 +12087,7 @@ dependencies = [ "snafu 0.8.6", "sql", "sqlparser", + "stacker", "store-api", "substrait 1.4.0-alpha.0", "table", diff --git a/src/datatypes/src/series/time.rs b/src/datatypes/src/series/time.rs index eae023a1fc3..bf5b50adf24 100644 --- a/src/datatypes/src/series/time.rs +++ b/src/datatypes/src/series/time.rs @@ -148,19 +148,41 @@ impl EvalGrid { self.timestamps.get_or_try_init(|| { let len = usize::try_from(self.spec.len) .map_err(|_| invalid_series("grid length overflows usize"))?; - let mut values = Vec::new(); - values - .try_reserve_exact(len) - .map_err(|_| invalid_series("grid timestamp allocation failed"))?; - for index in 0..len { - let timestamp = - (self.spec.start_ms as i128) + (index as i128) * (self.spec.step_ms as i128); - values.push(timestamp as i64); - } - Ok(values.into()) + Ok(self.gather_timestamps(0, len)?.into()) }) } + /// Whether the whole timestamp array exists already. + /// + /// A consumer that pays for the array -- an execution reserving what it is about to allocate -- + /// asks this before it materializes a grid: a grid that is already materialized was charged to + /// whoever materialized it, and a second consumer must not charge it again. + pub fn is_materialized(&self) -> bool { + self.timestamps.get().is_some() + } + + /// Returns the timestamps of the positions `start..start + len` as an array of their own, + /// leaving the grid unmaterialized. + /// + /// Every value is [`Self::timestamp_at`]'s, in position order, so gathering a window of the + /// grid costs its own length and not the length of the grid it is cut from. + fn gather_timestamps(&self, start: usize, len: usize) -> Result> { + let mut values = Vec::new(); + values + .try_reserve_exact(len) + .map_err(|_| invalid_series("grid timestamp allocation failed"))?; + let end = start.checked_add(len).context(InvalidSeriesSnafu { + reason: "grid range overflow".to_string(), + })?; + for position in start..end { + let Some(timestamp) = self.timestamp_at(position) else { + return invalid("grid range exceeds its grid"); + }; + values.push(timestamp); + } + Ok(values) + } + fn timestamp_at(&self, position: usize) -> Option { if position as u64 >= self.spec.len { return None; @@ -211,9 +233,22 @@ impl GridRange { pub fn is_empty(&self) -> bool { self.len == 0 } - /// Returns this range as a zero-copy Arrow buffer slice. + /// Returns the timestamps of the positions this range names. + /// + /// A range covering the whole grid shares the grid's array, which that grid has to materialize + /// for any reader of all of it. A range cut out of a larger grid computes the `len` timestamps + /// it names instead, each one [`EvalGrid::timestamp_at`]'s for its position: reading a window + /// out of a large grid costs the window's rows, and leaves a deferred grid deferred. pub fn timestamps(&self) -> Result> { - Ok(self.grid.timestamps()?.slice(self.start, self.len)) + if self.covers_grid() { + return Ok(self.grid.timestamps()?.clone()); + } + Ok(self.grid.gather_timestamps(self.start, self.len)?.into()) + } + + /// Whether this range names every position of its grid. + fn covers_grid(&self) -> bool { + self.start == 0 && u64::try_from(self.len).is_ok_and(|len| len == self.grid.spec.len) } } @@ -809,13 +844,20 @@ impl TimeAxis { /// Returns the timestamps of the carried rows, in compact-row order. /// - /// A single-run grid axis lends its buffer without copying; an axis of several runs, and a - /// chunked explicit axis, allocate one array of the carried rows. + /// An axis covering the whole grid lends the grid's array; a grid axis holding fewer positions + /// than its grid computes the rows it carries and lends nothing, so gathering a window out of + /// a large grid costs the window and leaves a deferred grid deferred. An axis of several runs, + /// and a chunked explicit axis, allocate one array of the carried rows. pub fn timestamps(&self) -> Result> { match self { - Self::Grid(range) => Ok(Cow::Borrowed( - &range.grid.timestamps()?[range.start()..range.start() + range.len()], - )), + Self::Grid(range) => { + if !range.covers_grid() { + return Ok(Cow::Owned( + range.grid.gather_timestamps(range.start(), range.len())?, + )); + } + Ok(Cow::Borrowed(&range.grid.timestamps()?[..])) + } Self::GridRanges(ranges) => { let mut timestamps = Vec::with_capacity(ranges.len()); for (position, _) in self.positions() { diff --git a/src/datatypes/src/series/time/tests.rs b/src/datatypes/src/series/time/tests.rs index 73e165f6f5f..33d5321a08d 100644 --- a/src/datatypes/src/series/time/tests.rs +++ b/src/datatypes/src/series/time/tests.rs @@ -118,11 +118,67 @@ fn deferred_grid_allocation_failure_is_reported_and_not_cached() { Arc::new(EvalGrid::try_deferred(GridSpec::try_new(i64::MIN, 1, len).unwrap()).unwrap()); assert!(grid.timestamps().is_err()); assert!(grid.timestamps.get().is_none()); - let range = GridRange::try_new(grid.clone(), 0, 1).unwrap(); - assert!(range.timestamps().is_err()); - assert!(TimeAxis::Grid(range).timestamps().is_err()); + // A range naming every point of that grid asks for the array that does not fit, and the + // failure is still the series error rather than an abort. + let whole = GridRange::try_new(grid.clone(), 0, len as usize).unwrap(); + assert!(whole.timestamps().is_err()); + assert!(TimeAxis::Grid(whole).timestamps().is_err()); assert!(grid.timestamps().is_err()); assert!(grid.timestamps.get().is_none()); + // A range that is nearly the whole grid gathers its own rows, and its allocation fails too: + // the gathered array is the range's, so the failure is reported rather than aborted. + let nearly = GridRange::try_new(grid.clone(), 0, len as usize - 1).unwrap(); + assert!(nearly.timestamps().is_err()); + assert!(TimeAxis::Grid(nearly).timestamps().is_err()); + assert!(grid.timestamps.get().is_none()); +} + +/// A window cut out of a large grid computes the rows it names instead of materializing the grid. +/// +/// The gathered values are the grid's own points, so a subrange reads the same timestamps it did +/// when it was a slice of the whole array; what changes is the cost, which is the window's rows +/// and no longer the grid's. +#[test] +fn subrange_timestamps_gather_only_the_rows_they_name() { + let len = (isize::MAX as u64 / std::mem::size_of::() as u64) + 1; + let grid = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(-5, 2, len).unwrap()).unwrap()); + let range = GridRange::try_new(grid.clone(), 7, 3).unwrap(); + assert!(!grid.is_materialized()); + let gathered = range.timestamps().unwrap(); + assert_eq!(&gathered[..], &[9, 11, 13]); + let axis = TimeAxis::Grid(range); + let axis_timestamps = axis.timestamps().unwrap(); + assert_eq!(axis_timestamps.as_ref(), &[9, 11, 13]); + // Every gathered point is what the axis answers for the row holding it. + for (row, timestamp) in axis_timestamps.iter().enumerate() { + assert_eq!(*timestamp, axis.timestamp(row).unwrap()); + } + assert!( + !grid.is_materialized(), + "the subrange materialized its grid" + ); + + // A range naming the whole grid still shares the grid's one array. + let small = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(10, 10, 4).unwrap()).unwrap()); + let all = GridRange::try_new(small.clone(), 0, 4).unwrap(); + assert!(!small.is_materialized()); + let shared = all.timestamps().unwrap(); + assert!( + small.is_materialized(), + "the whole-grid range did not materialize the grid it shares" + ); + assert_eq!(shared.as_ptr(), small.timestamps().unwrap().as_ptr()); + assert_eq!( + TimeAxis::Grid(all).timestamps().unwrap().as_ref(), + &[10, 20, 30, 40] + ); + + // An empty range at the end of a grid has no row to gather and materializes nothing. + let grid = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(0, 1, 3).unwrap()).unwrap()); + let empty = GridRange::try_new(grid.clone(), 3, 0).unwrap(); + assert!(empty.timestamps().unwrap().is_empty()); + assert!(TimeAxis::Grid(empty).timestamps().unwrap().is_empty()); + assert!(!grid.is_materialized()); } #[test] @@ -167,11 +223,20 @@ fn grid_materialization_revalidates_public_specs_and_shares_ranges_by_value() { let b = GridRange::try_new(Arc::new(EvalGrid::try_new(grid.spec()).unwrap()), 1, 3).unwrap(); assert_eq!(a, b); let values = a.timestamps().unwrap(); - assert_eq!( + // A subrange computes its own rows rather than slicing the grid's array, which it no longer + // shares: what it answers is still the grid's points, one per position it names. + assert_ne!( values.as_ptr(), - grid.timestamps().unwrap().as_ptr().wrapping_add(1) + grid.timestamps().unwrap().as_ptr().wrapping_add(1), + "the subrange borrowed the grid's array" ); assert_eq!(&values[..], &[-2, 1, 4]); + let whole = GridRange::try_new(grid.clone(), 0, 5).unwrap(); + assert_eq!( + whole.timestamps().unwrap().as_ptr(), + grid.timestamps().unwrap().as_ptr(), + "a whole-grid range copied the array it could share" + ); assert!(GridRange::try_new(grid.clone(), 5, 0).unwrap().is_empty()); assert!(GridRange::try_new(grid.clone(), 4, 2).is_err()); assert!(GridRange::try_new(grid, usize::MAX, 1).is_err()); diff --git a/src/mito2/src/series_index/maintenance.rs b/src/mito2/src/series_index/maintenance.rs index c127fd3a42d..d77e9551e59 100644 --- a/src/mito2/src/series_index/maintenance.rs +++ b/src/mito2/src/series_index/maintenance.rs @@ -226,6 +226,52 @@ pub(crate) async fn reconcile_series_indexes_with_mode( Ok(stats) } +/// How many sources one samples build reads at once. The writer bounds blocks and output sizes on +/// its own, so this only bounds the memory one pinned source group holds. +const SAMPLES_SOURCES_PER_BUILD: usize = 64; + +/// Splits uncovered sources into samples build groups, by UTC-day window and then by SST level. +/// +/// A samples artifact records the files it was built from and the days each of them contributed, so +/// a group is an identity, not a batch: every output of one group names the same sources, and +/// coverage is read back per group. Cutting the sources by file id alone - the previous behaviour - +/// put every level and every window into the same arbitrary chunks, so one new file re-cut them and +/// the days of unrelated levels were rebuilt. Grouping by window and level keeps a group's boundary +/// where the data puts it: a file of another level or another window joins its own group, and the +/// outputs already published for a group stay reusable when a source is added beside it. +/// +/// Windows sort before levels so the days of one window are built next to each other, and the files +/// inside a group are ordered by id so a build reads them in the same order every time. +fn sample_source_groups(files: &[FileHandle]) -> Vec> { + use std::collections::BTreeMap; + + use crate::series_index::builder::sample_day; + + let mut groups = BTreeMap::<(i64, i64, u8), Vec>::new(); + for file in files { + // The span the file covers is the window it writes into. A file holding several days is a + // window of its own here and still reports the days it actually filled. + let (start, end) = file.time_range(); + let (start, end) = (sample_day(start), sample_day(end)); + groups + .entry((start.min(end), start.max(end), file.level())) + .or_default() + .push(file.clone()); + } + groups + .into_values() + .map(|mut group| { + group.sort_unstable_by(|left, right| { + left.file_id() + .file_id() + .as_bytes() + .cmp(right.file_id().file_id().as_bytes()) + }); + group + }) + .collect() +} + /// Builds a changed snapshot, retaining reusable indexes and removing obsolete coverage. /// Returns `None` when no range or series indexes were added or removed. #[allow(clippy::too_many_arguments)] @@ -458,26 +504,27 @@ async fn build_index_version( .values() .flat_map(|handle| handle.entry().source_file_ids.iter().copied()) .collect(); - let mut uncovered: Vec<_> = files + let uncovered: Vec<_> = files .iter() .filter(|file| !covered.contains(&file.file_id().file_id())) .cloned() .collect(); - uncovered.sort_unstable_by_key(|file| file.file_id().file_id().to_string()); - for sources in uncovered.chunks(64) { - let handles = crate::series_index::builder::build_sample_windows( - store, - region, - metadata, - sources, - backend.as_ref(), - purger, - ) - .await?; - for handle in handles { - unpublished.1.push(handle.clone()); - stats.built_samples += 1; - daily.insert(handle.entry().file_id, handle); + for sources in sample_source_groups(&uncovered) { + for sources in sources.chunks(SAMPLES_SOURCES_PER_BUILD) { + let handles = crate::series_index::builder::build_sample_windows( + store, + region, + metadata, + sources, + backend.as_ref(), + purger, + ) + .await?; + for handle in handles { + unpublished.1.push(handle.clone()); + stats.built_samples += 1; + daily.insert(handle.entry().file_id, handle); + } } } stats.removed_samples += samples.keys().filter(|id| !daily.contains_key(id)).count(); @@ -693,13 +740,95 @@ fn publish_index_version(region: &MitoRegionRef, next: Arc) #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use common_time::Timestamp; use object_store::services::Memory; use super::*; + use crate::series_index::builder::sample_day; use crate::series_index::catalog::{RangeIndexEntry, SeriesIndexEntry}; use crate::series_index::purger::series_index_channel; + use crate::sst::file::{FileHandle, FileMeta}; + use crate::sst::file_purger::NoopFilePurger; + + /// A source file covering whole UTC days `start_day..=end_day` at `level`. + fn samples_source(id: u8, level: u8, start_day: i64, end_day: i64) -> FileHandle { + const DAY_MS: i64 = 86_400_000; + FileHandle::new( + FileMeta { + file_id: FileId::from_bytes([id; 16]), + time_range: ( + Timestamp::new_millisecond(start_day * DAY_MS), + Timestamp::new_millisecond(end_day * DAY_MS + DAY_MS - 1), + ), + level, + ..Default::default() + }, + Arc::new(NoopFilePurger), + ) + } + + /// Sources are grouped by the window they cover and by their level, not cut by file id. + /// + /// Every group is one window of one level, so a file that joins a level later cannot move the + /// boundary of a group built for another level, and the group's days are the days it covers. + #[test] + fn samples_sources_are_grouped_by_window_and_level() { + let mut files = vec![ + samples_source(1, 0, 0, 0), + samples_source(2, 0, 0, 0), + samples_source(3, 1, 0, 0), + samples_source(4, 0, 5, 5), + samples_source(5, 1, 5, 7), + samples_source(6, 0, 5, 5), + ]; + // Grouping must not depend on the ids or on the order the files arrive in. + files.rotate_left(2); + + let groups = sample_source_groups(&files); + let describe = |group: &[FileHandle]| { + let days: Vec<(i64, i64)> = group + .iter() + .map(|file| { + let (start, end) = file.time_range(); + (sample_day(start), sample_day(end)) + }) + .collect(); + (group[0].level(), days) + }; + assert_eq!( + vec![ + (0, vec![(0, 0), (0, 0)]), + (1, vec![(0, 0)]), + (0, vec![(5, 5), (5, 5)]), + (1, vec![(5, 7)]), + ], + groups + .iter() + .map(|group| describe(group)) + .collect::>(), + "one window of one level per group, ordered by window then level" + ); + // Every source is placed exactly once, and no group mixes a level or a window. + let mut grouped: Vec<_> = groups + .iter() + .flatten() + .map(|file| file.file_id().file_id().as_bytes().to_vec()) + .collect(); + let mut sources: Vec<_> = files + .iter() + .map(|file| file.file_id().file_id().as_bytes().to_vec()) + .collect(); + grouped.sort_unstable(); + sources.sort_unstable(); + assert_eq!(sources, grouped); + for group in &groups { + let (level, days) = describe(group); + assert!(group.iter().all(|file| file.level() == level)); + assert!(days.windows(2).all(|pair| pair[0] == pair[1])); + } + } #[rstest::rstest] fn test_prune_index_version( diff --git a/src/promql/src/aggregate.rs b/src/promql/src/aggregate.rs index 59ab98912aa..af4d8ad49d9 100644 --- a/src/promql/src/aggregate.rs +++ b/src/promql/src/aggregate.rs @@ -123,6 +123,24 @@ pub(crate) enum HistogramSamples { Folded, } +impl AggregateOp { + /// The words of `f64` one field's state of this aggregation keeps for one grid point. + /// + /// The last word is the presence word: the op's own count where its state carries one, so a + /// zero count reads as unseen, and a `0`/`1` flag where it does not. The words before it are + /// the components the op folds, in [`Self::state_fields`] order. A state therefore costs what + /// its own operation needs instead of what the widest operation needs: a block holds + /// `BLOCK_POINTS * fields * state_words * size_of::()` bytes however wide the op is. + pub(crate) const fn state_words(self) -> usize { + match self { + Self::Count | Self::Group => 1, + Self::Min | Self::Max => 2, + Self::Sum | Self::Stddev | Self::Stdvar => 3, + Self::Avg => 5, + } + } +} + /// What `op` does with a native histogram sample. pub(crate) fn histogram_samples(op: AggregateOp) -> HistogramSamples { match op { @@ -218,16 +236,45 @@ impl FloatAggregator { ) -> Result<()> { validate_aggregate_series(&self.schema, series, self.op)?; let existing_bytes = self.state_bytes(); - let grouping_workspace = grouping_workspace_bytes(&series.metric); - let mut workspace_bytes = - grouping_workspace.saturating_add(pending_bytes(self.schema.fields.len())); - budget(existing_bytes.saturating_add(workspace_bytes))?; - workspace_bytes = workspace_bytes.saturating_add( - self.labels - .len() - .saturating_mul(std::mem::size_of::<&str>()), - ); - budget(existing_bytes.saturating_add(workspace_bytes))?; + let group_capacity = self.groups.capacity(); + let map_capacity = self.group_index.capacity(); + // One admission for everything the fold can allocate before it writes a sample: the copy of + // this series' metric the grouping makes, the group's own copy of those labels and its + // key, the labels vector, and the group vector and index that may grow. What only a new + // group pays for is admitted whether or not this series opens one, so a chunk costs one + // pool call here instead of one per step, and a refusal still arrives before any of it is + // allocated. + // + // The grouping copy, the second workspace the copy is grouped into, and the group's + // retained labels and key are each a copy or a subset of this series' labels, so three + // copies of the one workspace bound all of them. + let label_workspace = grouping_workspace_bytes(&series.metric); + budget( + existing_bytes + .saturating_add(label_workspace.saturating_mul(3)) + .saturating_add(pending_bytes(self.schema.fields.len())) + .saturating_add( + self.labels + .len() + .saturating_mul(std::mem::size_of::<&str>()), + ) + // A realloc that cannot grow in place keeps both buffers alive while it copies, + // so a growing step admits the new buffer beside the old one, which the retained + // state already holds. The map is admitted the same way. + .saturating_add(if self.groups.len() == group_capacity { + group_capacity + .saturating_mul(2) + .max(4) + .saturating_mul(size_of::()) + } else { + 0 + }) + .saturating_add(if self.group_index.len() == map_capacity { + hash_map_backing_bytes(next_hash_capacity(map_capacity)) + } else { + 0 + }), + )?; let labels: Vec<&str> = self.labels.iter().map(|label| &**label).collect(); let grouping = if self.by { Grouping::By(&labels) @@ -235,51 +282,12 @@ impl FloatAggregator { Grouping::Without(&labels) }; let metric = grouping.apply(&series.metric)?; - let key_bytes = metric - .tags() - .len() - .saturating_add(1) - .saturating_mul(std::mem::size_of::<(Arc, Arc)>()); - workspace_bytes = workspace_bytes - .saturating_add(grouping_workspace_bytes(&metric)) - .saturating_add(key_bytes); - budget(existing_bytes.saturating_add(workspace_bytes))?; let key = label_key(&metric); let key_retained = key .capacity() .saturating_mul(size_of::<(Arc, Arc)>()); let new_group = !self.group_index.contains_key(&key); - let group_capacity = self.groups.capacity(); - let map_capacity = self.group_index.capacity(); let old_map_backing = hash_map_backing_bytes(map_capacity); - if new_group { - let next_group_capacity = if self.groups.len() == group_capacity { - group_capacity.saturating_mul(2).max(4) - } else { - group_capacity - }; - let group_realloc_bytes = if next_group_capacity > group_capacity { - next_group_capacity.saturating_mul(size_of::()) - } else { - 0 - }; - budget( - existing_bytes - .saturating_add(metric_bytes(&metric)) - .saturating_add(key_bytes) - // A realloc that cannot grow in place keeps both buffers alive while it - // copies, so a growing step admits the old buffer beside the new one -- - // the same reason the map below admits both tables. The old buffer alone - // is already in `existing_bytes`; what is added here is the new one. - .saturating_add(group_realloc_bytes) - .saturating_add(if self.group_index.len() == map_capacity { - hash_map_backing_bytes(next_hash_capacity(map_capacity)) - } else { - 0 - }) - .saturating_add(workspace_bytes), - )?; - } let group_capacity_growth = if new_group && self.groups.len() == group_capacity { group_capacity.saturating_mul(2).max(4) } else { @@ -375,7 +383,8 @@ impl FloatAggregator { .len() .saturating_mul(BLOCK_POINTS) .saturating_mul(self.schema.fields.len()) - .saturating_mul(std::mem::size_of::()), + .saturating_mul(self.op.state_words()) + .saturating_mul(std::mem::size_of::()), ) .saturating_add(btree_backing_bytes(group.blocks.len())) .saturating_add(histogram_backing_bytes(group.histograms.len())) @@ -459,6 +468,8 @@ fn finish_groups_budgeted( DataFusionError::Execution(format!("cannot reserve aggregate row: {error}")) })?; let mut completed_output_bytes = 0usize; + let words = op.state_words(); + let point_words = width.saturating_mul(words); for group in groups { // A group that folded histogram samples answers histogram columns; one that folded floats // keeps the float path below, so a group without a single histogram payload is built @@ -487,7 +498,7 @@ fn finish_groups_budgeted( } // One builder serves every group, so each result series starts explicitly. builder.begin_series(); - let (rows, runs) = group_rows_runs(&group, width); + let (rows, runs) = group_rows_runs(&group, width, op); let finish_bytes = finish_working_bytes(rows, runs, width); budget( state @@ -499,12 +510,16 @@ fn finish_groups_budgeted( builder.reserve_rows(rows)?; for (&block, states) in &group.blocks { // The last block can extend past the grid; only points with a sample are pushed. - for (offset, point) in states.chunks_exact(width).enumerate() { - if !point.iter().any(|state| state.seen) { + for (offset, point) in states.chunks_exact(point_words).enumerate() { + if !point.chunks_exact(words).any(|state| state_seen(state, op)) { continue; } row.clear(); - row.extend(point.iter().map(|state| state.finish(op))); + row.extend( + point + .chunks_exact(words) + .map(|state| state_finish(state, op)), + ); builder.push(block * BLOCK_POINTS + offset, &row)?; } } @@ -541,11 +556,12 @@ fn finish_histogram_group( // A field's column is one type. The fold already refuses a histogram where it had folded a // float at that position; a float folded after a histogram is caught here, where the column // type is chosen for the whole group. - if group - .blocks - .values() - .any(|states| states.iter().any(|state| state.seen)) - { + let words = op.state_words(); + if group.blocks.values().any(|states| { + states + .chunks_exact(words) + .any(|state| state_seen(state, op)) + }) { return Err(DataFusionError::Execution( "a native aggregate cannot mix float and histogram samples in one field".into(), )); @@ -697,16 +713,18 @@ fn finish_groups( finish_groups_budgeted(schema, op, groups, 0, |_| Ok(())) } -fn group_rows_runs(group: &Group, width: usize) -> (usize, usize) { +fn group_rows_runs(group: &Group, width: usize, op: AggregateOp) -> (usize, usize) { if width == 0 { return (0, 0); } + let words = op.state_words(); + let point_words = width * words; let mut rows = 0usize; let mut runs = 0usize; let mut previous: Option = None; for (&block, states) in &group.blocks { - for (offset, point) in states.chunks_exact(width).enumerate() { - if point.iter().any(|state| state.seen) { + for (offset, point) in states.chunks_exact(point_words).enumerate() { + if point.chunks_exact(words).any(|state| state_seen(state, op)) { let index = block.saturating_mul(BLOCK_POINTS).saturating_add(offset); rows = rows.saturating_add(1); if previous.is_none_or(|previous| previous.checked_add(1) != Some(index)) { @@ -1093,7 +1111,7 @@ fn next_hash_capacity(capacity: usize) -> usize { /// words; the leaf form is smaller than that. The header allowance is generous on purpose, so the /// bound does not depend on how a toolchain lays the node out. fn btree_node_bytes() -> usize { - 11 * size_of::<(usize, Box<[FloatState]>)>() + 16 * size_of::() + 11 * size_of::<(usize, Box<[f64]>)>() + 16 * size_of::() } /// An upper bound on the bytes a B-tree of `entries` holds. @@ -1148,11 +1166,21 @@ fn label_key(metric: &SeriesLabels) -> LabelKey { /// input never expands to the full grid and dense input costs one allocation per block. const BLOCK_POINTS: usize = 64; +/// Bytes one allocated block keeps: a state per point per field, [`AggregateOp::state_words`] +/// words of `f64` each. The fold, the merge and the recount all size a block through this, so what +/// is admitted and what is allocated cannot drift apart. +fn block_bytes(width: usize, op: AggregateOp) -> usize { + BLOCK_POINTS + .saturating_mul(width) + .saturating_mul(op.state_words()) + .saturating_mul(size_of::()) +} + struct Group { metric: SeriesLabels, /// Keyed by absolute grid index / `BLOCK_POINTS`; each block holds one state per field for - /// each of its points, point by point. - blocks: BTreeMap>, + /// each of its points, point by point, each state as many `f64` words as the op folds. + blocks: BTreeMap>, /// The histogram states of this group, keyed by absolute grid position and field. /// /// Only an aggregate that folds a payload allocates one, and only where it saw a histogram @@ -1227,6 +1255,20 @@ struct FieldFold { op: AggregateOp, } +impl FieldFold { + /// Words one point of this fold's block keeps: every field's state, side by side. + fn point_words(self) -> usize { + self.width.saturating_mul(self.op.state_words()) + } + + /// Where this field's state of the point at `offset` starts inside a block. + fn state_start(self, offset: usize) -> usize { + offset + .saturating_mul(self.point_words()) + .saturating_add(self.field_index.saturating_mul(self.op.state_words())) + } +} + impl Group { /// Folds one series' float samples into this group's blocks, one field at a time. /// @@ -1311,14 +1353,15 @@ impl Group { let block = index / BLOCK_POINTS; let (states, admitted) = self.block_states(block, fold, budget, retained)?; retained = admitted; - states[(index % BLOCK_POINTS) * fold.width + fold.field_index] - .update(fold.op, sample); + let words = fold.op.state_words(); + let start = fold.state_start(index % BLOCK_POINTS); + FloatState::new(&mut states[start..start + words], fold.op).update(sample); while let Some(&(at, sample)) = samples.peek() && at / BLOCK_POINTS == block { samples.next(); - states[(at % BLOCK_POINTS) * fold.width + fold.field_index] - .update(fold.op, sample); + let start = fold.state_start(at % BLOCK_POINTS); + FloatState::new(&mut states[start..start + words], fold.op).update(sample); } } } @@ -1336,13 +1379,11 @@ impl Group { fold: FieldFold, budget: &mut impl FnMut(usize) -> Result<()>, retained: usize, - ) -> Result<(&'a mut [FloatState], usize)> { + ) -> Result<(&'a mut [f64], usize)> { let new_block = !self.blocks.contains_key(&block); - let (block_bytes, backing_delta) = if new_block { + let (states_bytes, backing_delta) = if new_block { ( - BLOCK_POINTS - .saturating_mul(fold.width) - .saturating_mul(size_of::()), + block_bytes(fold.width, fold.op), btree_backing_bytes(self.blocks.len() + 1) .saturating_sub(btree_backing_bytes(self.blocks.len())), ) @@ -1351,13 +1392,14 @@ impl Group { }; let retained = retained .saturating_add(backing_delta) - .saturating_add(block_bytes); + .saturating_add(states_bytes); if new_block { budget(retained)?; } - let states = self.blocks.entry(block).or_insert_with(|| { - vec![FloatState::default(); BLOCK_POINTS * fold.width].into_boxed_slice() - }); + let states = self + .blocks + .entry(block) + .or_insert_with(|| vec![0.0; BLOCK_POINTS * fold.point_words()].into_boxed_slice()); Ok((&mut states[..], retained)) } } @@ -1387,8 +1429,10 @@ impl Group { // A field's column is either float or histogram, so a position that already folded a float // cannot also fold a histogram. The other order is caught when the fold is finished, which // is where the column type is chosen. + let words = fold.op.state_words(); + let start = fold.state_start(position % BLOCK_POINTS); if let Some(states) = self.blocks.get(&(position / BLOCK_POINTS)) - && states[(position % BLOCK_POINTS) * fold.width + fold.field_index].seen + && state_seen(&states[start..start + words], fold.op) { return Err(DataFusionError::Execution( "a native aggregate cannot mix float and histogram samples in one field".into(), @@ -1819,95 +1863,137 @@ fn histogram_backing_bytes(entries: usize) -> usize { entries.div_ceil(5).saturating_mul(histogram_node_bytes()) } -/// The float part of Prometheus' `groupedAggregation`. -#[derive(Clone, Copy, Default)] -struct FloatState { - seen: bool, - value: f64, - mean: f64, - compensation: f64, - count: f64, - incremental_mean: bool, +/// The float part of Prometheus' `groupedAggregation`, as a view of one state's words. +/// +/// A state is [`AggregateOp::state_words`] words of the block it sits in: the components the op +/// folds, then the presence word. The view reads and writes that layout in place, so folding +/// allocates nothing per state and an op that keeps less memory keeps less. +struct FloatState<'a> { + words: &'a mut [f64], + op: AggregateOp, } -impl FloatState { - fn update(&mut self, op: AggregateOp, sample: f64) { - if !self.seen { - *self = Self { - seen: true, - value: match op { - // The running sum of squared deviations; Prometheus starts it at NaN for a - // non-finite first sample. - AggregateOp::Stddev | AggregateOp::Stdvar if !sample.is_finite() => f64::NAN, - AggregateOp::Stddev | AggregateOp::Stdvar => 0.0, - _ => sample, - }, - mean: sample, - count: 1.0, - ..Self::default() - }; +/// Whether the state in `words` has seen a sample: its presence word is set. +/// +/// The presence word is the last one, which is the op's count where its state carries one - a +/// state that has folded nothing has no count - and a `0`/`1` flag where it does not. +fn state_seen(words: &[f64], op: AggregateOp) -> bool { + words[op.state_words() - 1] != 0.0 +} + +/// The value the state in `words` finishes to, or `None` if it has seen no sample. +fn state_finish(words: &[f64], op: AggregateOp) -> Option { + if !state_seen(words, op) { + return None; + } + Some(match op { + // `value`, its compensation, then the presence flag. + AggregateOp::Sum => words[0] + words[1], + // `value`, `compensation`, `mean`, the incremental flag, then the count. + AggregateOp::Avg if words[3] != 0.0 => words[2] + words[1], + AggregateOp::Avg => words[0] / words[4] + words[1] / words[4], + AggregateOp::Count => words[0], + AggregateOp::Min | AggregateOp::Max => words[0], + AggregateOp::Group => 1.0, + // `value`, `mean`, then the count. + AggregateOp::Stdvar => words[0] / words[2], + AggregateOp::Stddev => (words[0] / words[2]).sqrt(), + }) +} + +impl<'a> FloatState<'a> { + /// The state at `words`, which is `op`'s layout: [`AggregateOp::state_words`] words. + fn new(words: &'a mut [f64], op: AggregateOp) -> Self { + debug_assert_eq!(words.len(), op.state_words()); + Self { words, op } + } + + /// Folds one sample in, initializing the state if it had seen none. + fn update(self, sample: f64) { + let (op, words) = (self.op, self.words); + if !state_seen(words, op) { + match op { + // The count is the presence word. + AggregateOp::Count => words[0] = 1.0, + // Presence alone: `group` answers 1 for any kind of sample, NaN included, so the + // sample itself is not kept - a zero sample would read as unseen. + AggregateOp::Group => words[0] = 1.0, + // `value`, then the presence flag. + AggregateOp::Min | AggregateOp::Max => { + words[0] = sample; + words[1] = 1.0; + } + // `value`, its compensation, then the presence flag. + AggregateOp::Sum => { + words[0] = sample; + words[1] = 0.0; + words[2] = 1.0; + } + // The running sum of squared deviations, which Prometheus starts at NaN for a + // non-finite first sample; then `mean` and the count that says it is seen. + AggregateOp::Stddev | AggregateOp::Stdvar => { + words[0] = if sample.is_finite() { 0.0 } else { f64::NAN }; + words[1] = sample; + words[2] = 1.0; + } + // `value`, `compensation`, `mean`, the incremental flag, then the count. + AggregateOp::Avg => { + words[0] = sample; + words[1] = 0.0; + words[2] = sample; + words[3] = 0.0; + words[4] = 1.0; + } + } return; } match op { AggregateOp::Sum => { - (self.value, self.compensation) = kahan_inc(sample, self.value, self.compensation); + (words[0], words[1]) = kahan_inc(sample, words[0], words[1]); } - AggregateOp::Avg => self.update_avg(sample), - AggregateOp::Count => self.count += 1.0, + AggregateOp::Avg => update_avg(words, sample), + AggregateOp::Count => words[0] += 1.0, AggregateOp::Max => { - if self.value < sample || self.value.is_nan() { - self.value = sample; + if words[0] < sample || words[0].is_nan() { + words[0] = sample; } } AggregateOp::Min => { - if self.value > sample || self.value.is_nan() { - self.value = sample; + if words[0] > sample || words[0].is_nan() { + words[0] = sample; } } AggregateOp::Group => {} // Welford's update, uncompensated as in Prometheus. AggregateOp::Stddev | AggregateOp::Stdvar => { - self.count += 1.0; - let delta = sample - self.mean; - self.mean += delta / self.count; - self.value += delta * (sample - self.mean); + let count = words[2] + 1.0; + let delta = sample - words[1]; + words[1] += delta / count; + words[0] += delta * (sample - words[1]); + words[2] = count; } } } +} - /// Sums directly until the running sum would overflow, then switches to an incremental mean. - fn update_avg(&mut self, sample: f64) { - self.count += 1.0; - if !self.incremental_mean { - let (value, compensation) = kahan_inc(sample, self.value, self.compensation); - if !value.is_infinite() { - (self.value, self.compensation) = (value, compensation); - return; - } - self.incremental_mean = true; - self.mean = self.value / (self.count - 1.0); - self.compensation /= self.count - 1.0; +/// `avg`'s update: sums directly until the running sum would overflow, then switches to an +/// incremental mean, whose flag is the fourth word. +fn update_avg(words: &mut [f64], sample: f64) { + let count = words[4] + 1.0; + words[4] = count; + if words[3] == 0.0 { + let (value, compensation) = kahan_inc(sample, words[0], words[1]); + if !value.is_infinite() { + words[0] = value; + words[1] = compensation; + return; } - let q = (self.count - 1.0) / self.count; - (self.mean, self.compensation) = - kahan_inc(sample / self.count, q * self.mean, q * self.compensation); - } - - fn finish(self, op: AggregateOp) -> Option { - if !self.seen { - return None; - } - Some(match op { - AggregateOp::Sum => self.value + self.compensation, - AggregateOp::Avg if self.incremental_mean => self.mean + self.compensation, - AggregateOp::Avg => self.value / self.count + self.compensation / self.count, - AggregateOp::Count => self.count, - AggregateOp::Min | AggregateOp::Max => self.value, - AggregateOp::Group => 1.0, - AggregateOp::Stdvar => self.value / self.count, - AggregateOp::Stddev => (self.value / self.count).sqrt(), - }) + words[3] = 1.0; + words[2] = words[0] / (count - 1.0); + words[1] /= count - 1.0; } + let q = (count - 1.0) / count; + (words[2], words[1]) = kahan_inc(sample / count, q * words[2], q * words[1]); } /// Prometheus `kahansum.Inc`, with the Neumaier swap. diff --git a/src/promql/src/aggregate/state.rs b/src/promql/src/aggregate/state.rs index 92cb6e70ac1..de53c6b9feb 100644 --- a/src/promql/src/aggregate/state.rs +++ b/src/promql/src/aggregate/state.rs @@ -42,9 +42,9 @@ use datatypes::series::time::{GridSpan, TimeAxis}; use crate::aggregate::{ AggregateOp, BLOCK_POINTS, Counts, FloatAggregator, FloatState, Group, HistogramAccumulator, - HistogramEvidence, LabelKey, accumulator_bytes, btree_backing_bytes, count_series, + HistogramEvidence, LabelKey, accumulator_bytes, block_bytes, btree_backing_bytes, count_series, evidence_summary, finish_groups, finish_groups_budgeted, hash_map_backing_bytes, - histogram_backing_bytes, kahan_inc, label_key, metric_bytes, next_hash_capacity, + histogram_backing_bytes, kahan_inc, label_key, metric_bytes, next_hash_capacity, state_seen, }; use crate::map::validate_float_series; @@ -127,7 +127,7 @@ impl FloatAggregator { // export builds two one-row Arrow batches per point, which the chunk they are handed // to is charged for only once that chunk exists. let (points, payloads) = if group.histograms.is_empty() { - (seen_points(group, width), 0) + (seen_points(group, width, self.op), 0) } else { let payloads = group .histograms @@ -165,18 +165,21 @@ impl FloatAggregator { DataFusionError::Execution("aggregate state is indexed on an evaluated grid".into()) })?; let grid_len = grid.spec().len as usize; + let op = self.op; + let words = op.state_words(); + let point_words = width.saturating_mul(words); let mut groups: Vec = Vec::with_capacity(self.groups.len()); for group in self.groups { let points = if group.histograms.is_empty() { - seen_points(&group, width) + seen_points(&group, width, op) } else { // A field's state is one family, so a group that kept histogram payloads exports // those: a float state beside them would be one column carrying two kinds. - if group - .blocks - .values() - .any(|states| states.iter().any(|state| state.seen)) - { + if group.blocks.values().any(|states| { + states + .chunks_exact(words) + .any(|state| state_seen(state, op)) + }) { return Err(DataFusionError::Execution( "an aggregate field cannot carry both float and histogram state".into(), )); @@ -221,8 +224,8 @@ impl FloatAggregator { continue; } for (&block, block_states) in &group.blocks { - for (offset, point) in block_states.chunks_exact(width).enumerate() { - if !point.iter().any(|state| state.seen) { + for (offset, point) in block_states.chunks_exact(point_words).enumerate() { + if !point.chunks_exact(words).any(|state| state_seen(state, op)) { continue; } let position = block * BLOCK_POINTS + offset; @@ -232,8 +235,8 @@ impl FloatAggregator { ))); } extend_spans(&mut spans, position); - for (field, state) in point.iter().enumerate() { - states[field].push(state.partial().map(StatePayload::Float)); + for (field, state) in point.chunks_exact(words).enumerate() { + states[field].push(state_partial(state, op).map(StatePayload::Float)); } } } @@ -299,12 +302,14 @@ pub(super) fn exported_payload_bytes(accumulator: &HistogramAccumulator) -> usiz } /// The points of `group` that carry a state: the ones a partial of it holds. -fn seen_points(group: &Group, width: usize) -> usize { +fn seen_points(group: &Group, width: usize, op: AggregateOp) -> usize { + let words = op.state_words(); + let point_words = width.saturating_mul(words); group.blocks.values().fold(0usize, |count, states| { count.saturating_add( states - .chunks_exact(width) - .filter(|point| point.iter().any(|state| state.seen)) + .chunks_exact(point_words) + .filter(|point| point.chunks_exact(words).any(|state| state_seen(state, op))) .count(), ) }) @@ -681,9 +686,10 @@ impl FloatMerger { budget: &mut impl FnMut(usize) -> Result<()>, ) -> Result<()> { let width = self.schema.fields.len(); - let block_bytes = BLOCK_POINTS - .saturating_mul(width) - .saturating_mul(size_of::()); + let op = self.op; + let words = op.state_words(); + let point_words = width.saturating_mul(words); + let states_bytes = block_bytes(width, op); let blocks = &mut self.groups[index].blocks; let block_index = point / BLOCK_POINTS; let entries = blocks.len(); @@ -693,21 +699,24 @@ impl FloatMerger { budget( self.retained .saturating_add(live) - .saturating_add(block_bytes) + .saturating_add(states_bytes) .saturating_add(btree_backing_bytes(entries.saturating_add(1))), )?; self.retained = self .retained - .saturating_add(block_bytes) + .saturating_add(states_bytes) .saturating_add(btree_backing_bytes(entries.saturating_add(1))) .saturating_sub(btree_backing_bytes(entries)); - entry.insert(vec![FloatState::default(); BLOCK_POINTS * width].into_boxed_slice()) + entry.insert(vec![0.0; BLOCK_POINTS * point_words].into_boxed_slice()) } }; - let offset = (point % BLOCK_POINTS) * width; - for (slot, state) in block[offset..offset + width].iter_mut().zip(states) { + let offset = (point % BLOCK_POINTS) * point_words; + for (slot, state) in block[offset..offset + point_words] + .chunks_exact_mut(words) + .zip(states) + { if let Some(state) = state { - slot.merge(self.op, FloatState::from_partial(state)); + FloatState::new(slot, op).merge_partial(&state); } } Ok(()) @@ -906,97 +915,166 @@ fn state_at( Ok(Some(state)) } -impl FloatState { - fn partial(&self) -> Option { - self.seen.then_some(AccumulatorState { - value: self.value, - mean: self.mean, - compensation: self.compensation, - count: self.count, - incremental_mean: self.incremental_mean, - }) +/// The exported form of the state in `words`, or `None` if it has seen no sample. +/// +/// Only the components the op keeps are filled in; the rest stay at their defaults, exactly as the +/// frame that carries them leaves them out. +fn state_partial(words: &[f64], op: AggregateOp) -> Option { + if !state_seen(words, op) { + return None; } + Some(match op { + // `value`, its compensation, then the presence flag. + AggregateOp::Sum => AccumulatorState { + value: words[0], + compensation: words[1], + ..Default::default() + }, + // `value`, `compensation`, `mean`, the incremental flag, then the count. + AggregateOp::Avg => AccumulatorState { + value: words[0], + compensation: words[1], + mean: words[2], + incremental_mean: words[3] != 0.0, + count: words[4], + }, + AggregateOp::Count => AccumulatorState { + count: words[0], + ..Default::default() + }, + AggregateOp::Min | AggregateOp::Max => AccumulatorState { + value: words[0], + ..Default::default() + }, + AggregateOp::Group => AccumulatorState::default(), + // `value`, `mean`, then the count. + AggregateOp::Stddev | AggregateOp::Stdvar => AccumulatorState { + value: words[0], + mean: words[1], + count: words[2], + ..Default::default() + }, + }) +} - fn from_partial(state: AccumulatorState) -> Self { - Self { - seen: true, - value: state.value, - mean: state.mean, - compensation: state.compensation, - count: state.count, - incremental_mean: state.incremental_mean, +impl<'a> FloatState<'a> { + /// Writes `other` into this state, for a state that has seen nothing yet. + fn store_partial(self, other: &AccumulatorState) { + let (op, words) = (self.op, self.words); + match op { + AggregateOp::Count => words[0] = other.count, + // Presence alone, whatever the other side keeps: `group` answers 1 either way. + AggregateOp::Group => words[0] = 1.0, + AggregateOp::Min | AggregateOp::Max => { + words[0] = other.value; + words[1] = 1.0; + } + AggregateOp::Sum => { + words[0] = other.value; + words[1] = other.compensation; + words[2] = 1.0; + } + AggregateOp::Stddev | AggregateOp::Stdvar => { + words[0] = other.value; + words[1] = other.mean; + words[2] = other.count; + } + AggregateOp::Avg => { + words[0] = other.value; + words[1] = other.compensation; + words[2] = other.mean; + words[3] = if other.incremental_mean { 1.0 } else { 0.0 }; + words[4] = other.count; + } } } /// Folds the state of another partition into this one. - fn merge(&mut self, op: AggregateOp, other: Self) { - if !self.seen { - *self = other; + fn merge_partial(self, other: &AccumulatorState) { + if !state_seen(self.words, self.op) { + self.store_partial(other); return; } + let (op, words) = (self.op, self.words); match op { AggregateOp::Sum => { - (self.value, self.compensation) = add_compensated(*self, other); + (words[0], words[1]) = + add_compensated((words[0], words[1]), (other.value, other.compensation)); } - AggregateOp::Count => self.count += other.count, + AggregateOp::Count => words[0] += other.count, AggregateOp::Group => {} AggregateOp::Max => { - if self.value < other.value || self.value.is_nan() { - self.value = other.value; + if words[0] < other.value || words[0].is_nan() { + words[0] = other.value; } } AggregateOp::Min => { - if self.value > other.value || self.value.is_nan() { - self.value = other.value; + if words[0] > other.value || words[0].is_nan() { + words[0] = other.value; } } - AggregateOp::Avg => self.merge_avg(other), + AggregateOp::Avg => merge_avg(words, other), AggregateOp::Stddev | AggregateOp::Stdvar => { // Chan et al.'s pairwise update. `delta * (delta * weight)` keeps the product // finite where it is: `delta²` alone overflows for `[0]` and `[1.5e154]`, whose // merged sum of squares is finite. - let count = self.count + other.count; - let delta = other.mean - self.mean; - let weight = self.count * other.count / count; - self.mean += delta * (other.count / count); - self.value = self.value + other.value + delta * (delta * weight); - self.count = count; + let count = words[2] + other.count; + let delta = other.mean - words[1]; + let weight = words[2] * other.count / count; + words[1] += delta * (other.count / count); + words[0] = words[0] + other.value + delta * (delta * weight); + words[2] = count; } } } +} - /// Adds the sums while they stay finite, and otherwise continues with the count-weighted - /// mean, as the serial update switches to an incremental mean once its sum overflows. - fn merge_avg(&mut self, other: Self) { - let count = self.count + other.count; - if !self.incremental_mean && !other.incremental_mean { - let (value, compensation) = add_compensated(*self, other); - if !value.is_infinite() { - self.value = value; - self.compensation = compensation; - self.count = count; - return; - } +/// Adds the sums while they stay finite, and otherwise continues with the count-weighted mean, as +/// the serial update switches to an incremental mean once its sum overflows. +/// +/// The state words are `value`, `compensation`, `mean`, the incremental flag and the count. +fn merge_avg(words: &mut [f64], other: &AccumulatorState) { + let count = words[4] + other.count; + if words[3] == 0.0 && !other.incremental_mean { + let (value, compensation) = + add_compensated((words[0], words[1]), (other.value, other.compensation)); + if !value.is_infinite() { + words[0] = value; + words[1] = compensation; + words[4] = count; + return; } - let (mean, compensation) = self.mean_form(); - let (other_mean, other_compensation) = other.mean_form(); - let (weight, other_weight) = (self.count / count, other.count / count); - (self.mean, self.compensation) = kahan_inc( - other_mean * other_weight, - mean * weight, - compensation * weight + other_compensation * other_weight, - ); - self.incremental_mean = true; - self.count = count; } + let (mean, compensation) = mean_form(words[0], words[1], words[2], words[3] != 0.0, words[4]); + let (other_mean, other_compensation) = mean_form( + other.value, + other.compensation, + other.mean, + other.incremental_mean, + other.count, + ); + let (weight, other_weight) = (words[4] / count, other.count / count); + (words[2], words[1]) = kahan_inc( + other_mean * other_weight, + mean * weight, + compensation * weight + other_compensation * other_weight, + ); + words[3] = 1.0; + words[4] = count; +} - /// The mean and its compensation, converting a sum as the serial update does on switching. - fn mean_form(&self) -> (f64, f64) { - if self.incremental_mean { - (self.mean, self.compensation) - } else { - (self.value / self.count, self.compensation / self.count) - } +/// The mean and its compensation, converting a sum as the serial update does on switching. +fn mean_form( + value: f64, + compensation: f64, + mean: f64, + incremental: bool, + count: f64, +) -> (f64, f64) { + if incremental { + (mean, compensation) + } else { + (value / count, compensation / count) } } @@ -1004,9 +1082,9 @@ impl FloatState { /// Kahan addends: adding the compensation plainly would drop what the compensation already /// holds once later partials cancel the sums, as merging `[1e100, 1]`, `[-1e100, 1e84]` and /// `[-1e84]` would lose the `1`. -fn add_compensated(this: FloatState, other: FloatState) -> (f64, f64) { - let (value, compensation) = kahan_inc(other.value, this.value, this.compensation); - kahan_inc(other.compensation, value, compensation) +fn add_compensated(this: (f64, f64), other: (f64, f64)) -> (f64, f64) { + let (value, compensation) = kahan_inc(other.0, this.0, this.1); + kahan_inc(other.1, value, compensation) } /// Merges `count_values` results of several partitions. Each partial already carries its full diff --git a/src/promql/src/aggregate/tests.rs b/src/promql/src/aggregate/tests.rs index 76f12d99734..3cd19c84dab 100644 --- a/src/promql/src/aggregate/tests.rs +++ b/src/promql/src/aggregate/tests.rs @@ -616,6 +616,96 @@ fn incremental_retained_bytes_match_full_recount_for_many_groups() { } } +/// A state keeps the components its operation folds plus a presence word, which the count already +/// provides where the state has one. That is the whole width of a state, so a `max` state costs two +/// words where an `avg` state costs five instead of every state costing one fixed struct. +#[test] +fn a_state_is_as_wide_as_the_operation_it_belongs_to() { + for (op, words) in [ + (AggregateOp::Count, 1), + (AggregateOp::Group, 1), + (AggregateOp::Min, 2), + (AggregateOp::Max, 2), + (AggregateOp::Sum, 3), + (AggregateOp::Stddev, 3), + (AggregateOp::Stdvar, 3), + (AggregateOp::Avg, 5), + ] { + assert_eq!(op.state_words(), words, "{op:?}"); + let counted = op + .state_fields() + .iter() + .any(|field| matches!(field, StateField::Count)); + assert_eq!( + op.state_words(), + op.state_fields().len() + usize::from(!counted), + "{op:?} keeps its own components and nothing else" + ); + } +} + +/// What a block keeps is what its operation folds: for every operation the fold's retained figure +/// and the recount of its live blocks agree, and a block holds one state per point per field, each +/// as many words as the operation keeps. +#[test] +fn every_operation_recounts_the_states_it_keeps() { + let schema = schema_with_grid_len(&["value", "other"], 256); + for op in [ + AggregateOp::Sum, + AggregateOp::Count, + AggregateOp::Avg, + AggregateOp::Min, + AggregateOp::Max, + AggregateOp::Group, + AggregateOp::Stddev, + AggregateOp::Stdvar, + ] { + let mut aggregator = + FloatAggregator::try_new(&schema, op, Grouping::By(&["zone"])).unwrap(); + assert_eq!( + aggregator.state_bytes(), + aggregator.recounted_state_bytes(), + "{op:?} initial state" + ); + let updates = [ + input( + &schema, + metric("m", &[("zone", "a")]), + 0, + &[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]], + ), + input( + &schema, + metric("m", &[("zone", "a")]), + 64, + &[vec![Some(4.0)], vec![Some(5.0)]], + ), + input( + &schema, + metric("m", &[("zone", "b")]), + 128, + &[vec![Some(6.0)], vec![Some(7.0)]], + ), + ]; + for (index, series) in updates.iter().enumerate() { + aggregator.update(series).unwrap(); + assert_eq!( + aggregator.state_bytes(), + aggregator.recounted_state_bytes(), + "{op:?} update {index}" + ); + } + // Three blocks of two fields, each state as wide as the operation folds it. + let blocks = &aggregator.groups[0].blocks; + assert_eq!(blocks.len(), 2, "{op:?} blocks the first group wrote"); + assert_eq!( + blocks.values().map(|block| block.len()).sum::(), + blocks.len() * BLOCK_POINTS * schema.fields.len() * op.state_words(), + "{op:?} words per block" + ); + } +} + /// The unbudgeted entry point is the budgeted one with a no-op budget, so this compares one path /// with itself: it is a smoke check that the wrapper still runs, not evidence that the kernels are /// right. The numbers are checked against Prometheus in `aggregates_match_prometheus` and @@ -679,9 +769,7 @@ fn adjacent_blocks_are_admitted_and_accounted_separately() { assert_eq!(aggregator.groups[0].blocks.len(), 2); // A new block is admitted on the retained state alone, so the last admission settles at // exactly what the fold keeps and no staging buffer is charged for the samples in it. - let block = BLOCK_POINTS - .saturating_mul(schema.fields.len()) - .saturating_mul(std::mem::size_of::()); + let block = block_bytes(schema.fields.len(), AggregateOp::Sum); let (first, second) = ( admissions[admissions.len() - 2], admissions[admissions.len() - 1], @@ -710,9 +798,7 @@ fn a_rejected_block_allocation_does_not_insert_a_block() { // The figure a new block's admission adds: the block's states and its tree node, on top of // whatever the fold already holds. Located by the value, not by which call reaches it, so // reordering the earlier admissions cannot make this test pass without testing anything. - let block_states = BLOCK_POINTS - .saturating_mul(schema.fields.len()) - .saturating_mul(std::mem::size_of::()); + let block_states = block_bytes(schema.fields.len(), AggregateOp::Sum); let block = block_states.saturating_add(btree_backing_bytes(1)); let mut aggregator = FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap(); @@ -756,9 +842,7 @@ fn a_rejected_later_block_keeps_the_blocks_the_group_already_holds() { // The group and the tree node are already there, so the block's own states are the whole // charge of the second block. let later = input(&schema, metric("m", &[]), 64, &[vec![Some(2.0)]]); - let block_states = BLOCK_POINTS - .saturating_mul(schema.fields.len()) - .saturating_mul(std::mem::size_of::()); + let block_states = block_bytes(schema.fields.len(), AggregateOp::Sum); let threshold = retained .saturating_add(btree_backing_bytes(2).saturating_sub(btree_backing_bytes(1))) .saturating_add(block_states); @@ -864,9 +948,7 @@ fn the_fold_releases_the_series_workspace_before_it_admits_a_block() { let state = aggregator.state_bytes(); assert_eq!(aggregator.recounted_state_bytes(), state); - let block = BLOCK_POINTS - .saturating_mul(schema.fields.len()) - .saturating_mul(std::mem::size_of::()); + let block = block_bytes(schema.fields.len(), AggregateOp::Sum); let entry = state .saturating_sub(block) .saturating_sub(btree_backing_bytes(1)); @@ -885,6 +967,105 @@ fn the_fold_releases_the_series_workspace_before_it_admits_a_block() { ); } +/// A chunk admits its workspace and the group it may open in one pool call, releases the +/// workspace at the fold entry, and pays for each block it opens: three calls for a chunk that +/// falls in one block, four for one that opens two. +/// +/// The fold used to admit the workspace, then a wider bound of it, then the grouping and its key, +/// then the new group on top of them: six admissions for a chunk that opens a group and one block, +/// seven for one that opens two, on top of the runtime's own resize once the chunk is folded. +#[test] +fn a_chunk_admits_its_workspace_release_and_blocks_in_three_or_four_calls() { + let schema = schema(&["value", "other"]); + let one_block = input( + &schema, + metric("m", &[("zone", "a")]), + 0, + &[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]], + ); + // Two samples of the first field straddle the edge at 64, so the fold opens a second block. + let two_blocks = input( + &schema, + metric("m", &[("zone", "a")]), + 63, + &[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]], + ); + let admissions = |series: &Series| { + let mut aggregator = + FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone"])).unwrap(); + let mut calls = Vec::new(); + aggregator + .update_budgeted(series, |bytes| { + calls.push(bytes); + Ok(()) + }) + .unwrap(); + assert_eq!(aggregator.state_bytes(), aggregator.recounted_state_bytes()); + calls + }; + + let one = admissions(&one_block); + assert_eq!( + one.len(), + 3, + "the workspace, the release and one block: {one:?}" + ); + assert!( + one[0] > one[1], + "the workspace was admitted beside the retained state: {one:?}" + ); + let block = block_bytes(schema.fields.len(), AggregateOp::Sum); + assert_eq!( + one[2], + one[1] + .saturating_add(block) + .saturating_add(btree_backing_bytes(1)), + "the block is admitted on the retained state alone" + ); + + let two = admissions(&two_blocks); + assert_eq!( + two.len(), + 4, + "the workspace, the release and two blocks: {two:?}" + ); + assert!(two[3] > two[2], "each block is admitted separately"); +} + +/// A chunk that only writes into states its group already holds costs the workspace admission and +/// the release, and no block. +#[test] +fn a_chunk_of_existing_blocks_admits_only_its_workspace_and_release() { + let schema = schema(&["value"]); + let mut aggregator = + FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone"])).unwrap(); + aggregator + .update(&input( + &schema, + metric("m", &[("zone", "a")]), + 0, + &[vec![Some(1.0)]], + )) + .unwrap(); + let mut calls = Vec::new(); + aggregator + .update_budgeted( + &input( + &schema, + metric("m", &[("zone", "a")]), + 1, + &[vec![Some(2.0)]], + ), + |bytes| { + calls.push(bytes); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(calls.len(), 2, "the workspace and the release: {calls:?}"); + assert_eq!(aggregator.state_bytes(), aggregator.recounted_state_bytes()); +} + /// A block a field holds no value in is not allocated: two fields whose samples fall in different /// blocks leave the group with the blocks they write, not the blocks between them. #[test] @@ -1983,7 +2164,7 @@ fn merge_charges_one_block_per_new_block_not_per_point() { aggregator.update(&series).unwrap(); let partial = aggregator.finish_partial().unwrap(); assert_eq!(partial.groups[0].states[0].len(), 64); - let block = BLOCK_POINTS * 64 * std::mem::size_of::(); + let block = block_bytes(64, AggregateOp::Sum); let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); let (peak, result) = peak_admission(|budget| merge_partial_budgeted(&mut merger, partial, budget)); diff --git a/src/promql/src/extension_plan.rs b/src/promql/src/extension_plan.rs index 71674ac5fbf..fc619cd7047 100644 --- a/src/promql/src/extension_plan.rs +++ b/src/promql/src/extension_plan.rs @@ -44,8 +44,8 @@ pub use empty_metric::{EmptyMetric, EmptyMetricExec, EmptyMetricStream, build_sp pub use fused_series::is_canonical_producer_exec; pub use fused_series::{ FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec, - FusedSeriesStream, InputLocality, input_locality, matching_operands_drop_key, - tree_source_fields, + FusedSeriesStream, InputLocality, input_locality, keys_survive_metadata_drop, + matching_operands_drop_key, tree_source_fields, }; pub use histogram_fold::{ HistogramFold, HistogramFoldExec, HistogramFoldOperation, HistogramFoldStream, diff --git a/src/promql/src/extension_plan/fused_series.rs b/src/promql/src/extension_plan/fused_series.rs index 7774117b51c..c3305d5567f 100644 --- a/src/promql/src/extension_plan/fused_series.rs +++ b/src/promql/src/extension_plan/fused_series.rs @@ -78,6 +78,7 @@ use datafusion::physical_plan::{ Partitioning, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datatypes::data_type::DataType as _; +use datatypes::schema::ColumnSchema; use datatypes::series::SeriesOrdering; use datatypes::series::frame::{ FrameContent, FrameLabelMode, SeriesFrameDeclaration, aggregate_state_frame_schema, @@ -96,7 +97,7 @@ use crate::extension_plan::series_repartition::SeriesRepartitionExec; use crate::extension_plan::{RawSeriesSourceExec, SeriesRouting}; use crate::grid_series::METADATA_LABELS; use crate::histogram_eval::HISTOGRAM_ROUTING_EXCLUDED_LABELS; -use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor}; +use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor, SourceFacts}; use crate::native::metrics::{NodeMetricsSnapshot, is_node_metric}; use crate::native::plan::{ AggregateStage, BoundMap, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree, @@ -990,6 +991,7 @@ impl FusedSeries { output_schema, self.binding.time_index.clone(), tags, + self.binding.metric_name.clone(), self.format, target_partitions, self.canonical_producer, @@ -2125,6 +2127,13 @@ pub struct FusedSeriesExec { time_column: String, /// The tag column names, indexed by [`OutputColumn::Tag`]. tags: Arc<[String]>, + /// The one metric every input series is named, taken from the binding. + /// + /// The canonical pass judged a dropped label set by this: a node that drops the metric name is + /// only told to stream when everything below it is named the same one. The execution reads it + /// so the collection it performs at run time is decided by that same proof; `None` is a binding + /// that names no metric - one raw source per input - and every consumer then collects. + metric_name: Option>, /// The columns this node produces. output_schema: SchemaRef, /// The evidence claimed by this plan's output partitions. @@ -2172,20 +2181,42 @@ struct StoredSeriesLabels { /// Every tag name a stored series reports besides its metric name. tags: Arc<[Arc]>, /// Whether the primary key keeps telling two stored series apart once a name-dropping kernel - /// has normalized their labels. - /// - /// Two facts make that true. The tags and the primary key must be free of the metadata labels: - /// `__type__` and `__unit__` go with the metric name when a kernel drops it, so a binding that - /// keys a series by either of them stops telling two series apart once the name is gone. And - /// every primary-key column must always decode a value, which is what a non-nullable column - /// states. A kernel that drops the name drops every label whose value is empty with it, and a - /// nullable column lets one series hold `NULL` while another holds `''`: two different storage - /// keys, which decode to an absent label and to an empty one, so both are handed over as no - /// label at all once the empty one is dropped. Two such series that agree on every other tag - /// then leave as one identity, whether they arrive in one partition or in two. + /// has normalized their labels; see [`keys_survive_metadata_drop`] for the rule itself. keys_survive_metadata_drop: bool, } +/// Whether the storage key of a scan keeps telling its stored series apart once a name-dropping +/// kernel has normalized their labels. +/// +/// `logical_tags` are the tag names every stored series of the scan reports, and `key_columns` are +/// the columns the scan keys one series by. Two facts make the drop injective, and both are read +/// from the scan's own binding rather than from a tree above it: +/// +/// - Neither the tags nor the key columns may be a metadata label. `__type__` and `__unit__` go with +/// the metric name when a kernel drops it, so a scan that keys a series by either of them stops +/// telling two series apart once the name is gone. +/// - Every key column must always decode a value, which is what a non-nullable column states. A +/// kernel that drops the name drops every label whose value is empty with it, and a nullable +/// column lets one series hold `NULL` while another holds `''`: two different storage keys, which +/// decode to an absent label and to an empty one, so both are handed over as no label at all once +/// the empty one is dropped. Two such series that agree on every other tag then leave as one +/// identity, whether they arrive in one partition or in two. +/// +/// This is the rule the logical canonical pass applies to decide whether a node that drops the +/// metadata labels has to be collected, and the one the physical per-series scan applies to decide +/// whether such a node can stream. Both read it here so the two answers cannot drift apart: the +/// planner reads it from the resolved table's metadata, the execution from the bound scan. +pub fn keys_survive_metadata_drop<'a, 'b>( + logical_tags: impl IntoIterator, + key_columns: impl IntoIterator, +) -> bool { + let metadata = |name: &str| METADATA_LABELS.contains(&name); + logical_tags.into_iter().all(|tag| !metadata(tag)) + && key_columns + .into_iter() + .all(|column| !metadata(&column.name) && !column.is_nullable()) +} + impl StoredSeriesLabels { /// The labels `label_source` proves about every stored series it reads. fn of(label_source: &RawLabelSource) -> Option { @@ -2199,19 +2230,17 @@ impl StoredSeriesLabels { else { return None; }; - let metadata = |name: &str| METADATA_LABELS.contains(&name); Some(Self { tags: logical_tags .iter() .map(|tag| Arc::clone(&tag.name)) .collect(), - keys_survive_metadata_drop: logical_tags.iter().all(|tag| !metadata(&tag.name)) - && physical_metadata.primary_key_columns().all(|column| { - // A nullable key column can be `NULL` in one series and `''` in another, and a - // name-dropping kernel turns both into no label; only a column that always - // decodes a value keeps the two apart under such a kernel. - !metadata(&column.column_schema.name) && !column.column_schema.is_nullable() - }), + keys_survive_metadata_drop: keys_survive_metadata_drop( + logical_tags.iter().map(|tag| tag.name.as_ref()), + physical_metadata + .primary_key_columns() + .map(|column| &column.column_schema), + ), }) } @@ -2490,6 +2519,7 @@ impl FusedSeriesExec { output_schema, time_column, tags, + None, FusedOutputFormat::PlainRows, None, false, @@ -2504,6 +2534,7 @@ impl FusedSeriesExec { output_schema: SchemaRef, time_column: String, tags: Vec, + metric_name: Option>, format: FusedOutputFormat, matching_parallelism: Option, canonical_producer: bool, @@ -2787,6 +2818,7 @@ impl FusedSeriesExec { classification: classification.into(), time_column, tags: tags.into(), + metric_name, output_schema, duplicate_evidence: Mutex::new(EvidenceClaims { evidence: None, @@ -2938,6 +2970,7 @@ impl ExecutionPlan for FusedSeriesExec { self.output_schema.clone(), self.time_column.clone(), self.tags.to_vec(), + self.metric_name.clone(), self.format, self.matching_parallelism, self.canonical_producer, @@ -2970,10 +3003,19 @@ impl ExecutionPlan for FusedSeriesExec { evidence: Arc::clone(evidence), active: true, }); - let mut state = match NativeExecutionState::try_new( + // The canonical pass decided where a merge belongs from the labels the binding names; the + // execution judges the collection it performs by the same fact, so an already-streaming + // window is not turned into a collecting one here. + let sources = SourceFacts::uniform( + self.tags.to_vec(), + self.metric_name.clone(), + self.series_schemas.len(), + ); + let mut state = match NativeExecutionState::try_new_with_sources( &self.tree, Arc::clone(&context.runtime_env().memory_pool), evidence.clone(), + &sources, ) { Ok(state) => state, Err(error) => { diff --git a/src/promql/src/extension_plan/fused_series/tests.rs b/src/promql/src/extension_plan/fused_series/tests.rs index 9b5c7a02000..182270b0067 100644 --- a/src/promql/src/extension_plan/fused_series/tests.rs +++ b/src/promql/src/extension_plan/fused_series/tests.rs @@ -238,6 +238,24 @@ fn window_tree_of(function: RangeFunction) -> NativeTree { NativeTree::try_new(root).expect("a window function keeps the raw input and its grid") } +/// `rate(x[5m])` as the canonical pass hands it over: one merge over the window's output. +/// +/// A range function drops the metric name, so two series that agreed on every other label leave the +/// window under one label set. Collecting them is what refuses a second value at one position: the +/// merge groups by the label set the window handed out and reports the collision, as Prometheus +/// refuses two series of one label set. +fn canonical_rate_tree_of(schema: Arc) -> NativeTree { + let window = rate_tree_of(schema); + let output = window.root().output.clone(); + NativeTree::try_new(NativeNode { + expr: NativeExpr::Merge { + input: Box::new(window.root().clone()), + }, + output, + }) + .expect("a merge over a window on a grid is a legal tree") +} + /// `left + right` on one grid, both operands reading the one actual source. fn binary_tree(left_schema: Arc, right_schema: Arc) -> NativeTree { binary_tree_over_sources(BinaryOp::Add, left_schema, right_schema, 0, 0) @@ -1945,7 +1963,11 @@ fn zone_output_schema() -> SchemaRef { /// `rate(x[5m])` over frames that carry `zone` as well as `host`, with both in the output. fn zone_node() -> FusedSeries { - let tree = rate_tree(); + zone_node_with(rate_tree()) +} + +/// The same node over `tree`, so a test can stage the tree the canonical pass produces. +fn zone_node_with(tree: NativeTree) -> FusedSeries { let binding = binding_of( &tree, raw_schema(), @@ -3397,7 +3419,7 @@ fn rows(batches: &[RecordBatch]) -> Vec { const DUPLICATE_LABEL_SET: &str = "vector cannot contain metrics with the same labelset {host=\"web-1\"}"; -/// Two raw series that are already one Prometheus identity are refused by the runtime that would +/// Two raw series that are already one Prometheus identity are refused by the merge that would /// have to answer both. /// /// `{host="web-1", zone=""}` and `{host="web-1"}` are one identity to Prometheus: an empty label @@ -3405,12 +3427,16 @@ const DUPLICATE_LABEL_SET: &str = /// What the two *raw* keys differ in is the empty `zone` against the absent one, which the scan /// encodes differently and the decoder therefore opens two input series for. `rate` drops the /// metric name and the empty value, so the second raw series arrives on the canonical label set the -/// first one already occupies; the refusal names that label set rather than answering it, because -/// answering would report one point twice. +/// first one already occupies, and the merge collecting the window's output refuses it: the +/// refusal names that label set rather than answering it, because answering would report one point +/// twice. #[tokio::test] async fn differing_raw_keys_that_close_on_one_label_set_are_refused() { - let exec = plan_of(zone_node(), memory_exec(frames(&zone_chunks()))) - .expect("the runtime executes a rate tree"); + let exec = plan_of( + zone_node_with(canonical_rate_tree_of(raw_schema())), + memory_exec(frames(&zone_chunks())), + ) + .expect("the runtime executes a canonical rate tree"); let error = collect(exec, SessionContext::default().task_ctx()) .await .expect_err("the second raw key cannot be answered under the first one's label set"); @@ -3424,13 +3450,25 @@ async fn differing_raw_keys_that_close_on_one_label_set_are_refused() { /// /// The decoder holds the batch it is reading, the chunks it decoded from it and its own source, so a /// stream that kept its input after the error would hold all three until the consumer drops it. The -/// collision above is the error: it happens on the second series, after the first one's rows have -/// been handed out. +/// error staged here is the truncated second series: the first one's rows are handed out, and the +/// second one ends the input without saying it was its last chunk, which the frame decoder refuses. #[tokio::test] async fn a_runtime_error_after_rows_releases_the_input() { let dropped = Arc::new(AtomicBool::new(false)); let schema = frame_schema(); - let batches = frames(&zone_chunks()); + // The second series is cut off: its chunk does not close it. + let mut chunks = zone_chunks(); + chunks.pop().expect("the collision stages two series"); + let samples = series_samples(1.0); + chunks.push(chunk_of( + &raw_schema(), + "zone-absent", + metric(&[(HOST, "web-1")], Some(METRIC)), + ×_of(&samples), + vec![values_of(&samples)], + false, + )); + let batches = frames(&chunks); let source = staged(schema.clone(), { let dropped = Arc::clone(&dropped); move || Box::pin(StagedStream::new(schema.clone(), batches.clone()).dropped(&dropped)) @@ -3452,7 +3490,7 @@ async fn a_runtime_error_after_rows_releases_the_input() { } }; assert!( - error.to_string().contains(DUPLICATE_LABEL_SET), + error.to_string().contains("ended before key was final"), "refused for another reason: {error}" ); // The first series' rows were handed out before the error, and the input is gone although the @@ -3467,16 +3505,17 @@ async fn a_runtime_error_after_rows_releases_the_input() { ); } -/// A collision produced inside one native execution is still refused by its local runtime state. +/// A collision produced inside one native execution is refused by the merge that collects it. /// /// This is the collision the canonical label set exists for, at the layer that sees it: /// `http_requests_total{host="web-1"}` and `http_errors_total{host="web-1"}` reach the window as two /// identities and leave it as `{host="web-1"}`. [`FusedSeries`] reads one metric per scan, so this /// collision cannot be staged through it; the runtime is driven directly, with the names on the raw -/// chunks. +/// chunks. The window is handed over collected, as the canonical pass places it: nothing is answered +/// until its input ends, and the merge then refuses the second value of the one label set. #[test] fn two_differently_named_series_collide_once_rate_drops_the_name() { - let tree = rate_tree(); + let tree = canonical_rate_tree_of(raw_schema()); let mut execution = NativeExecutor::new( NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None) .expect("the runtime executes a rate tree"), @@ -3495,13 +3534,6 @@ fn two_differently_named_series_collide_once_rate_drops_the_name() { ), ) .expect("the first series is a legal raw chunk"); - assert!( - execution - .next_chunk() - .expect("the first series runs") - .is_some(), - "the first series is answered" - ); execution .push_raw( InputId::new(0), @@ -3515,8 +3547,11 @@ fn two_differently_named_series_collide_once_rate_drops_the_name() { ), ) .expect("the second series is a legal raw chunk"); + // Both series sit in the merge's collection: the second one puts a value where the first one + // already holds one, and the merge refuses the pair as it collects them. let error = execution - .next_chunk() + .end_of_input() + .and_then(|()| execution.next_chunk().map(|_| ())) .expect_err("the second series puts a value where the first one already holds one"); assert!( error.to_string().contains(DUPLICATE_LABEL_SET), diff --git a/src/promql/src/extension_plan/series_repartition.rs b/src/promql/src/extension_plan/series_repartition.rs index 7b23e22aa2c..974ff75287c 100644 --- a/src/promql/src/extension_plan/series_repartition.rs +++ b/src/promql/src/extension_plan/series_repartition.rs @@ -94,6 +94,8 @@ //! until its per-input close/EOF barrier is satisfied, then //! reserve merge workspace for that identity alone. This bounds merge workspace to one pending //! identity, while its inputs remain charged until handoff. No task waits for memory admission. +//! An output registers one consumer with the pool and splits every identity's charge off it, so the +//! pool holds an entry per output stream rather than one per series that stream routes. //! //! An output ends only when every input partition finished and its queue was drained; a chain that //! closed locally is not the end of anything. Each input sends its own EOF to each live output before @@ -2107,6 +2109,14 @@ struct RepartitionStream { inputs: usize, output: VecDeque, handoff: Arc>, + /// What every identity of this output is charged to. + /// + /// One registration for the whole output stream: each identity's descriptor, each owner's + /// merge workspace and each raw owner's reconstruction are reservations split off it, so the + /// pool holds an entry per output partition instead of one per series the output routes. A + /// reservation split off it still keeps its bytes live until it is dropped, and the pool sees + /// each of those bytes exactly once. + groups_reservation: MemoryReservation, /// Whether a new pass over the pending identities is owed after an input EOF. /// /// A chain can only complete the identity it carries, so the chain arm checks that group @@ -2190,6 +2200,7 @@ impl RepartitionStream { inputs, output: VecDeque::new(), handoff: Arc::new(Mutex::new(Handoff::default())), + groups_reservation: MemoryConsumer::new("SeriesRepartitionGroups").register(pool), rescan: false, resume_after: None, reported: false, @@ -2207,7 +2218,7 @@ impl RepartitionStream { let rows = group.fragments.iter().fold(0usize, |rows, series| { rows.saturating_add(series.time.len()) }); - let reservation = MemoryConsumer::new("SeriesRepartitionOwner").register(&self.pool); + let reservation = self.groups_reservation.new_empty(); reservation.try_grow(merge_workspace_bytes( rows, group.fragments.len(), @@ -2281,7 +2292,7 @@ impl RepartitionStream { // Reconstruction's working set depends on how the pieces interleave, which is only known // while it runs: charge each allocation as it is made rather than reserving a worst case // that a well-ordered identity never approaches. - let workspace = MemoryConsumer::new("SeriesRepartitionRawOwner").register(&self.pool); + let workspace = self.groups_reservation.new_empty(); let mut reserve = |bytes: usize| workspace.try_grow(bytes); let mut shrink = |bytes: usize| { let result = workspace.try_shrink(bytes); @@ -2480,8 +2491,7 @@ impl Stream for RepartitionStream { } }; if !this.groups.contains_key(&key) { - let metadata = - MemoryConsumer::new("SeriesRepartitionIdentity").register(&this.pool); + let metadata = this.groups_reservation.new_empty(); let key_bytes = match &key { OwnerKey::Evaluated((name, tags)) => { diff --git a/src/promql/src/extension_plan/series_repartition/tests.rs b/src/promql/src/extension_plan/series_repartition/tests.rs index 7e4e150931e..ec4efae6b4c 100644 --- a/src/promql/src/extension_plan/series_repartition/tests.rs +++ b/src/promql/src/extension_plan/series_repartition/tests.rs @@ -521,13 +521,77 @@ impl ExecutionPlan for Scripted { /// A task context whose memory pool is `pool`, so what the node charges can be read afterwards. fn task_context(pool: &Arc) -> Arc { + context_over(Arc::clone(pool) as Arc) +} + +/// A task context over `pool`, for a test that keeps its own pool to read it afterwards. +fn context_over(pool: Arc) -> Arc { let runtime = RuntimeEnvBuilder::new() - .with_memory_pool(Arc::clone(pool) as Arc) + .with_memory_pool(pool) .build() .expect("the runtime environment builds"); Arc::new(TaskContext::default().with_runtime(Arc::new(runtime))) } +/// A pool that counts the consumers registered with it, so a test can see how many entries the node +/// adds whatever the input carries. +#[derive(Debug)] +struct RegistrationCountingPool { + inner: GreedyMemoryPool, + registrations: AtomicUsize, +} + +impl RegistrationCountingPool { + fn new(bytes: usize) -> Arc { + Arc::new(Self { + inner: GreedyMemoryPool::new(bytes), + registrations: AtomicUsize::new(0), + }) + } + + /// How many consumers were registered with the pool, in total. + fn registrations(&self) -> usize { + self.registrations.load(Ordering::SeqCst) + } +} + +impl fmt::Display for RegistrationCountingPool { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "registration counting test pool") + } +} + +impl MemoryPool for RegistrationCountingPool { + fn name(&self) -> &str { + "registration counting test pool" + } + + fn register(&self, consumer: &MemoryConsumer) { + self.registrations.fetch_add(1, Ordering::SeqCst); + self.inner.register(consumer); + } + + fn unregister(&self, consumer: &MemoryConsumer) { + self.inner.unregister(consumer); + } + + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.inner.grow(reservation, additional); + } + + fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { + self.inner.shrink(reservation, shrink); + } + + fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> DataFusionResult<()> { + self.inner.try_grow(reservation, additional) + } + + fn reserved(&self) -> usize { + self.inner.reserved() + } +} + /// The node over `child` routing into `outputs` partitions. fn node(child: Arc, outputs: usize) -> Arc { Arc::new(SeriesRepartitionExec::try_new(child, outputs).expect("the node is built")) @@ -2692,6 +2756,42 @@ async fn single_fragment_large_identity_fits_finite_payload_budget() { ); } +/// Each identity used to register a pool consumer of its own, so the pool's bookkeeping grew with +/// the number of series an output routed. The stream registers once per output stream now, and the +/// identity's descriptor, an owner's merge workspace and a raw owner's reconstruction are +/// reservations split off it: more identities must not mean more entries, and the bytes each of +/// them holds are still released. +#[tokio::test] +async fn pool_entries_follow_output_streams_not_identities() { + let registrations = |identities: usize| async move { + let steps: Vec = (0..identities) + .map(|index| Step::chunk(chunk(labels(&format!("h{index:03}")), 1, 1, true))) + .collect(); + let pool = RegistrationCountingPool::new(1 << 20); + let context = context_over(Arc::clone(&pool) as Arc); + let plan: Arc = node(Scripted::plan(vec![steps]), 1); + let outputs = drain(&plan, &context).await; + let chunks = outputs + .into_iter() + .flat_map(|output| output.expect("the output routes every identity")) + .collect::>(); + assert_eq!(identities, chunks.len(), "each identity is published once"); + assert_eq!(0, pool.reserved(), "every charge is released with the read"); + pool.registrations() + }; + + let few = registrations(1).await; + let many = registrations(16).await; + assert_eq!( + few, many, + "the entries the node registers follow the identities: {few} for one and {many} for sixteen" + ); + assert!( + few <= 4, + "{few} pool entries for one input and one output stream" + ); +} + /// An input EOF already consumed still counts when a different input first contributes an identity. #[tokio::test] async fn eof_before_first_identity_contribution_makes_group_ready() { @@ -4188,6 +4288,83 @@ async fn owner_rejects_conflicting_values_at_same_identity_and_position() { ); } +/// The owner merges the histogram pieces of one label set that its inputs hold. +/// +/// A canonical boundary hands one label set to a single owner, and that owner is where the pieces the +/// producer's partitions collected meet: here two inputs hold complementary positions of one label +/// set and each piece carries its own histogram payload, so the merge rebuilds one field over the +/// union of the positions with every row still naming the payload it came from. The pieces are closed +/// as they arrive, so the merged series is also what proves the closing chunk of a chain survives the +/// merge. +#[tokio::test] +async fn owner_merges_histogram_pieces_of_one_label_set_across_inputs() { + let metric = labels("histogram-pieces"); + let (first, closing) = histogram_rows_chunk_with_sum(metric.clone(), 0, 1, 10.0); + let (second, _) = histogram_rows_chunk_with_sum(metric.clone(), 8, 1, 20.0); + let child = Scripted::plan(vec![ + vec![Step::Chunk(first, closing)], + vec![Step::chunk((second, true))], + ]); + let plan: Arc = node(child, 2); + let outputs = drain(&plan, &SessionContext::default().task_ctx()).await; + let owner = Routing::new(2).output(&metric); + let chunks = outputs[owner].as_ref().expect("the owner output decodes"); + assert_output_contract(chunks, &[(metric.clone(), vec![0, 8])]); + assert_eq!(1, chunks.iter().filter(|(_, closed)| *closed).count()); + // One row per valued position, in order, each holding the payload of the piece that pushed it: a + // merge that rebuilt the field without reindexing the payloads would answer one payload twice. + let merged = chunks + .iter() + .flat_map(|(series, _)| { + let field = series.fields.get(0).expect("one field"); + series.time.positions().map(move |(position, row)| { + let datatypes::series::samples::SampleRef::Histogram(view) = field.sample(row) + else { + panic!("position {position} of the merged series is not a histogram"); + }; + (position, view.sum, view.count) + }) + }) + .collect::>(); + assert_eq!( + vec![(0, 10.0, 4.0), (8, 20.0, 4.0)], + merged, + "the merged histograms are not the ones the pieces carried" + ); +} + +/// Two histogram pieces of one label set at one position fail the merge. +/// +/// The pieces were routed to one owner because they are one label set, so a position both of them +/// fill would carry two values of one field. The kernel reports that the way it does for floats, and +/// the histogram payload makes no difference to it. +#[tokio::test] +async fn owner_rejects_two_histogram_pieces_at_one_position() { + let metric = labels("histogram-conflict"); + let (first, _) = histogram_rows_chunk_with_sum(metric.clone(), 4, 1, 10.0); + let (second, _) = histogram_rows_chunk_with_sum(metric.clone(), 4, 1, 20.0); + let child = Scripted::plan(vec![ + vec![Step::chunk((first, true))], + vec![Step::chunk((second, true))], + ]); + let plan: Arc = node(child, 1); + let outputs = drain(&plan, &SessionContext::default().task_ctx()).await; + let error = outputs[0] + .as_ref() + .err() + .unwrap_or_else(|| { + panic!( + "two histogram pieces of one label set at one position were merged: {:?}", + outputs[0].as_ref().map(|chunks| summaries(chunks)) + ) + }) + .to_string(); + assert!( + error.contains("cannot contain metrics with the same labelset"), + "the failure does not name the shared label set: {error}" + ); +} + /// Same-identity input chunks are collected and emitted as one closed series. #[tokio::test] async fn same_canonical_key_chains_merge_into_one_series() { diff --git a/src/promql/src/grid_series.rs b/src/promql/src/grid_series.rs index 532283528dd..d3f754a7335 100644 --- a/src/promql/src/grid_series.rs +++ b/src/promql/src/grid_series.rs @@ -233,13 +233,7 @@ impl GridSeriesBuilder { for (field, value) in self.fields.iter_mut().zip(values) { match value { Some(GridSample::Histogram(h)) => { - field.values.push(0.0); - field.nulls.append(false); - let rows = field.values.len() - 1; - let column = field.histograms.get_or_insert_with(|| HistogramColumn { - batch: HistogramBatchBuilder::new(), - indices: vec![None; rows], - }); + let column = field.start_histogram(); column.indices.push(Some(h.push_to(&mut column.batch)?)); } Some(GridSample::Float(value)) => field.push_float(Some(*value)), @@ -249,6 +243,38 @@ impl GridSeriesBuilder { Ok(()) } + /// Like [`GridSeriesBuilder::push_samples`], with the values taken as the fields hand them out. + /// + /// A histogram value goes into its column straight from the view it was read as: the owned + /// form beside this builder exists for the paths that compute a histogram, while a selection + /// that only hands a sample on has nothing to compute and copies nothing. The row this writes + /// and the row [`GridSeriesBuilder::push_samples`] writes are the same, so a value pushed + /// either way leaves the same payload. + pub(crate) fn push_refs(&mut self, index: usize, values: &[SampleRef<'_>]) -> Result<()> { + if !self.start_row( + index, + values.len(), + values + .iter() + .all(|value| matches!(value, SampleRef::Missing)), + )? { + return Ok(()); + } + for (field, value) in self.fields.iter_mut().zip(values) { + match value { + SampleRef::Histogram(view) => { + let column = field.start_histogram(); + column + .indices + .push(Some(column.batch.push(*view).map_err(external)?)); + } + SampleRef::Float(value) => field.push_float(Some(*value)), + SampleRef::Missing => field.push_float(None), + } + } + Ok(()) + } + /// Validates the row and retains it; returns whether it holds a value to write. fn start_row(&mut self, index: usize, len: usize, empty: bool) -> Result { if len != self.schema.fields.len() { @@ -404,6 +430,18 @@ impl FieldBuilder { column.indices.push(None); } } + + /// Writes the value columns of a row holding a histogram, opening the histogram column if this + /// is its first value, and hands back the column the new row's index still goes into. + fn start_histogram(&mut self) -> &mut HistogramColumn { + self.values.push(0.0); + self.nulls.append(false); + let rows = self.values.len() - 1; + self.histograms.get_or_insert_with(|| HistogramColumn { + batch: HistogramBatchBuilder::new(), + indices: vec![None; rows], + }) + } } /// Prometheus `schema.IsMetadataLabel`: the labels removed with the metric name. diff --git a/src/promql/src/grid_series/tests.rs b/src/promql/src/grid_series/tests.rs index 98c65bcb16f..203de9e9f2c 100644 --- a/src/promql/src/grid_series/tests.rs +++ b/src/promql/src/grid_series/tests.rs @@ -16,7 +16,9 @@ use std::mem::size_of; use std::sync::Arc; use datafusion::arrow::array::{Float64Array, UInt32Array}; -use datatypes::series::histogram::{HistogramBatch, HistogramBatchBuilder}; +use datatypes::series::histogram::{ + CounterResetHint, HistogramBatch, HistogramBatchBuilder, HistogramView, +}; use datatypes::series::samples::{ DenseSamplesRef, FieldSpec, HistogramSamples, SampleRef, Samples, Series, SeriesFields, SeriesSchema, @@ -299,6 +301,130 @@ fn merge_keeps_histograms_and_rejects_any_doubled_field() { } } +/// The histogram payloads of one label set merge by position, whatever table each shard filed its +/// own payload in. +/// +/// A label set reaches a merge as one fragment per producer, and every fragment was built by its own +/// builder: each holds its own histogram table and its own indices into it, so the merged field has +/// to be rebuilt over the union of the positions with each row still naming the payload it was +/// pushed with. The fragments here are complementary, the position neither of two fragments holds +/// stays a hole, and a fragment that pushes nothing adds no row. +#[test] +fn merge_keeps_each_shards_histogram_payload() { + // One histogram per shard, each with its own count, sum and bucket payload: `count/sum` are the + // two fields a reader of the merged series checks a payload by. + let payloads = [ + OwnedHistogram::parse( + "0/0/0/3ff0000000000000/3ff0000000000000/1/0_2/4000000000000000,4008000000000000///", + ), + OwnedHistogram::parse("0/0/0/4008000000000000/4014000000000000/1/1_1/4010000000000000///"), + OwnedHistogram::parse("0/0/0/4014000000000000/4018000000000000/1/0_1/4014000000000000///"), + ]; + // The `(sum, count)` each payload was written with, in payload order. + let scalars = [(1.0, 1.0), (5.0, 3.0), (6.0, 5.0)]; + let schema = grid_schema(1, 8); + let shard = |rows: &[(usize, usize)]| { + let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap(); + for &(point, payload) in rows { + let histogram = FloatHistogram::try_from_view(&payloads[payload].view()).unwrap(); + builder + .push_samples(point, &[Some(GridSample::Histogram(Box::new(histogram)))]) + .unwrap(); + } + builder + .finish(metric(0)) + .unwrap() + .expect("the shard holds the rows it pushed") + }; + // A shard of the same label set holding nothing finishes with no series at all, so it adds + // neither a row nor a gap to what the others pushed. + assert!( + GridSeriesBuilder::try_new(schema.clone()) + .unwrap() + .finish(metric(0)) + .unwrap() + .is_none(), + "an empty shard handed out a series" + ); + let merged = merge_same_labelset( + &schema, + vec![shard(&[(0, 0), (3, 1)]), shard(&[(1, 1)]), shard(&[(4, 2)])], + ) + .unwrap(); + assert_eq!(merged.len(), 1, "one label set, so one merged series"); + let field = merged[0].fields.get(0).expect("one field"); + let merged_rows = merged[0] + .time + .positions() + .map(|(point, row)| { + let SampleRef::Histogram(view) = field.sample(row) else { + panic!("position {point} of the merged series is not a histogram"); + }; + let histogram = FloatHistogram::try_from_view(&view).unwrap(); + (point, histogram.sum, histogram.count) + }) + .collect::>(); + // One row per valued position, in order: position 2 is a hole in every shard and stays a hole, + // and each row carries the payload its own shard pushed. + assert_eq!( + vec![ + (0, scalars[0].0, scalars[0].1), + (1, scalars[1].0, scalars[1].1), + (3, scalars[1].0, scalars[1].1), + (4, scalars[2].0, scalars[2].1), + ], + merged_rows + ); + // The merged field is one freshly built table over those rows: the rows the shards held are + // addressed by the merged series' own indices, so a row cannot name a shard's table. + let SampleRef::Histogram(first) = field.sample(0) else { + panic!("the merged field holds no histogram"); + }; + assert_eq!(&[2.0, 3.0], first.positive.buckets); + let SampleRef::Histogram(second) = field.sample(1) else { + panic!("the merged field holds no histogram"); + }; + assert_eq!(&[4.0], second.positive.buckets); + assert_eq!(1, second.positive.span_lengths.len()); + assert_eq!(1, second.positive.span_offsets[0]); +} + +/// Two shards of one label set holding a histogram at one position are refused. +/// +/// The merge collects the pieces of one label set into one series, so a position two pieces fill +/// would carry two values of one field: that is the collision Prometheus reports rather than a +/// choice for this node to make. The refusal names the shared label set, as it does for floats. +#[test] +fn merge_rejects_a_histogram_position_two_shards_hold() { + let payloads = [ + OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/1/0_1/4000000000000000///"), + OwnedHistogram::parse("0/0/0/4000000000000000/4000000000000000/1/0_1/4000000000000000///"), + ]; + let schema = grid_schema(1, 8); + let shard = |point: usize, payload: usize| { + let histogram = FloatHistogram::try_from_view(&payloads[payload].view()).unwrap(); + let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap(); + builder + .push_samples(point, &[Some(GridSample::Histogram(Box::new(histogram)))]) + .unwrap(); + builder.finish(metric(0)).unwrap().unwrap() + }; + // Complementary positions merge whatever payloads they hold, so the only reason the second pair + // is refused is the position both of them fill. + let merged = merge_same_labelset(&schema, vec![shard(0, 0), shard(1, 1)]).unwrap(); + assert_eq!(1, merged.len()); + assert_eq!( + 2, + merged[0].time.len(), + "two shards holding different positions merge" + ); + let error = merge_same_labelset(&schema, vec![shard(1, 0), shard(1, 1)]).unwrap_err(); + assert!( + error.to_string().contains("same labelset"), + "the refusal does not name the shared label set: {error}" + ); +} + /// Two values a million positions apart are one series with two rows, addressed by two runs: the /// positions between them cost neither a row nor a value buffer sized by the span. #[test] @@ -1461,3 +1587,122 @@ mod acceptance { assert_eq!(read(&three_chunks), read(&two_chunks)); } } + +/// The payload of one histogram, field by field, with every float compared by its bits so that a +/// value that moved by one bit is not equal. +fn payload(view: &HistogramView<'_>) -> Vec { + let hint = match view.reset_hint { + CounterResetHint::Unknown => 0, + CounterResetHint::CounterReset => 1, + CounterResetHint::NotCounterReset => 2, + CounterResetHint::Gauge => 3, + CounterResetHint::Unrecognized(value) => (value as u64) | (1 << 32), + }; + let mut bits = vec![ + view.schema as u64, + hint, + view.start_timestamp.map_or(u64::MAX, |start| start as u64), + view.zero_threshold.to_bits(), + view.zero_count.to_bits(), + view.count.to_bits(), + view.sum.to_bits(), + view.custom_values.len() as u64, + ]; + bits.extend(view.custom_values.iter().map(|value| value.to_bits())); + for side in [&view.positive, &view.negative] { + bits.push(side.span_offsets.len() as u64); + bits.push(side.span_lengths.len() as u64); + bits.push(side.buckets.len() as u64); + bits.extend(side.span_offsets.iter().map(|&offset| offset as u64)); + bits.extend(side.span_lengths.iter().map(|&length| length as u64)); + bits.extend(side.buckets.iter().map(|bucket| bucket.to_bits())); + } + bits +} + +/// A value pushed from the view it was read as lands in the result as the payload the owned route +/// writes, start timestamp included: a selection that only hands a sample on leaves it as it is. +#[test] +fn a_pushed_view_writes_the_payload_of_the_owned_route() { + let schema = grid_schema(2, 3); + let histograms = [ + OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///"), + OwnedHistogram::parse("0/0/0/4000000000000000/4008000000000000/1/0_1/4000000000000000///"), + ]; + // The sample a scan read: the payload of its row plus the start timestamp the row carried. + let mut stamped = histograms[1].view(); + stamped.start_timestamp = Some(1_500); + let views = [histograms[0].view(), stamped]; + + let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap(); + builder + .push_samples( + 0, + &[ + Some(GridSample::Histogram(Box::new( + FloatHistogram::try_from_view(&views[0]).unwrap(), + ))), + Some(GridSample::Float(2.0)), + ], + ) + .unwrap(); + builder + .push_samples( + 2, + &[ + None, + Some(GridSample::Histogram(Box::new( + FloatHistogram::try_from_view(&views[1]).unwrap(), + ))), + ], + ) + .unwrap(); + let owned = builder.finish(metric(0)).unwrap().unwrap(); + + let mut builder = GridSeriesBuilder::try_new(schema).unwrap(); + builder + .push_refs(0, &[SampleRef::Histogram(views[0]), SampleRef::Float(2.0)]) + .unwrap(); + builder + .push_refs(2, &[SampleRef::Missing, SampleRef::Histogram(views[1])]) + .unwrap(); + let by_view = builder.finish(metric(0)).unwrap().unwrap(); + + let described = |series: &Series, field: usize| -> Vec<(usize, String, Option>)> { + let samples = &series.fields.as_slice()[field]; + series + .time + .positions() + .map(|(point, row)| match samples.sample(row) { + SampleRef::Missing => (point, "missing".to_string(), None), + SampleRef::Float(value) => { + (point, "float".to_string(), Some(vec![value.to_bits()])) + } + SampleRef::Histogram(view) => { + (point, "histogram".to_string(), Some(payload(&view))) + } + }) + .collect() + }; + for field in 0..2 { + assert_eq!( + described(&owned, field), + described(&by_view, field), + "field {field} came back different through the view route" + ); + } + assert_eq!( + described(&by_view, 1), + vec![ + (0, "float".to_string(), Some(vec![2.0f64.to_bits()])), + (2, "histogram".to_string(), Some(payload(&views[1]))), + ], + "the pushed view lost the payload it was read with" + ); + // The comparison covers the start timestamp, which the two routes had to preserve. + assert_ne!( + payload(&views[1]), + payload(&histograms[1].view()), + "the start timestamp is part of what is compared" + ); +} diff --git a/src/promql/src/histogram_eval/aggregate.rs b/src/promql/src/histogram_eval/aggregate.rs index 5583be6ecec..2c730d7e43a 100644 --- a/src/promql/src/histogram_eval/aggregate.rs +++ b/src/promql/src/histogram_eval/aggregate.rs @@ -15,6 +15,7 @@ //! `sum` and `avg` of native histograms within one aggregation group, following the histogram //! paths of Prometheus `aggregation`. +use std::borrow::Cow; use std::sync::Arc; use datafusion::arrow::array::UInt32Array; @@ -139,7 +140,7 @@ fn aggregate_point<'a>( has_float = true; None } - SampleRef::Histogram(view) => Some(FloatHistogram::try_from_view(&view)), + SampleRef::Histogram(view) => Some(FloatHistogram::try_from_view(&view).map(Cow::Owned)), }); let (value, _) = sum_or_avg(op, histograms.by_ref())?; // A fold stopped by mixed bucket kinds still reports conversion errors of later samples. @@ -159,12 +160,17 @@ fn aggregate_point<'a>( /// time, so the first error ends the fold. The result is derived, so it carries no start /// timestamp. /// +/// The histograms are carried by reference: the accumulator is the only value the fold copies +/// out, and every sample after the first is only read. A window of `w` samples therefore costs +/// one deep copy, not `w`, and a caller that already owns its window pays none for the samples it +/// keeps. +/// /// The flag reports whether any add had to reconcile two different custom-bucket sets, which is /// what Prometheus annotates with an info. It is carried out of the fold rather than recomputed: /// the fold already calls [`FloatHistogram::kahan_add`] and receives that outcome. -pub(crate) fn sum_or_avg( +pub(crate) fn sum_or_avg<'a>( op: HistogramAggregate, - histograms: impl IntoIterator>, + histograms: impl IntoIterator>>, ) -> Result<(Option, bool)> { let mut value: Option = None; let mut compensation: Option = None; @@ -175,23 +181,25 @@ pub(crate) fn sum_or_avg( let h = h?; count += 1.0; let Some(current) = value.as_mut() else { - value = Some(h); + // The one copy the fold takes: the accumulator starts as the first sample. + value = Some(h.into_owned()); continue; }; + let h = h.as_ref(); // Go drops the result with a warning for this addition error only; others propagate. if current.uses_custom_buckets() != h.uses_custom_buckets() { return Ok((None, custom_reconciled)); } match op { HistogramAggregate::Sum => { - let (c, outcome) = current.kahan_add(&h, compensation.take())?; + let (c, outcome) = current.kahan_add(h, compensation.take())?; custom_reconciled |= outcome.custom_bounds_reconciled; compensation = Some(c); } HistogramAggregate::Avg => { if mean.is_none() { let mut sum = current.clone(); - let (c, outcome) = sum.kahan_add(&h, compensation.clone())?; + let (c, outcome) = sum.kahan_add(h, compensation.clone())?; custom_reconciled |= outcome.custom_bounds_reconciled; if !sum.has_overflow() { *current = sum; @@ -210,7 +218,9 @@ pub(crate) fn sum_or_avg( if let Some(c) = compensation.as_mut() { c.mul(q); } - let mut to_add = h; + // The incremental mean scales the sample it adds, so this is the one branch that + // needs the histogram by value on the sample after the first. + let mut to_add = h.clone(); to_add.div(count); let mean = mean.as_mut().expect("set above"); mean.mul(q); diff --git a/src/promql/src/histogram_eval/float_histogram.rs b/src/promql/src/histogram_eval/float_histogram.rs index 1dd0b6e87f5..dd13215fae7 100644 --- a/src/promql/src/histogram_eval/float_histogram.rs +++ b/src/promql/src/histogram_eval/float_histogram.rs @@ -31,8 +31,46 @@ use crate::histogram_eval::native::{CUSTOM_BUCKETS_SCHEMA, bound}; type Buckets = Vec<(i32, f64)>; +/// How often a histogram of this thread was copied or built from a view, so a test can hold a path +/// to the copies it should not make rather than to the shape of its arguments. Test-only: a copy +/// and a borrow produce the same value, so a test comparing only values could not tell the two +/// apart. Both counters belong to the calling thread, so tests running side by side cannot see +/// each other's work. +#[cfg(test)] +pub(crate) mod copies { + use std::cell::Cell; + + thread_local! { + /// Deep copies [`super::FloatHistogram::clone`] made on this thread. + static CLONES: Cell = const { Cell::new(0) }; + /// Owned histograms [`super::FloatHistogram::try_from_view`] built on this thread. + static MATERIALIZED: Cell = const { Cell::new(0) }; + } + + /// Records one deep copy. + pub(crate) fn record_clone() { + CLONES.with(|clones| clones.set(clones.get() + 1)); + } + + /// Records one owned histogram built from a view. + pub(crate) fn record_materialized() { + MATERIALIZED.with(|built| built.set(built.get() + 1)); + } + + /// The deep copies made on this thread since the last call, which starts a new count. + pub(crate) fn take_clones() -> usize { + CLONES.with(|clones| clones.replace(0)) + } + + /// The owned histograms built from a view on this thread since the last call, which starts a + /// new count. + pub(crate) fn take_materialized() -> usize { + MATERIALIZED.with(|built| built.replace(0)) + } +} + /// A native histogram with absolute bucket counts. -#[derive(Debug, Clone)] +#[derive(Debug)] pub(crate) struct FloatHistogram { pub(crate) reset_hint: CounterResetHint, /// The start timestamp of the sample this histogram was read from, in milliseconds. @@ -74,8 +112,30 @@ pub(crate) struct AddOutcome { pub(crate) custom_bounds_reconciled: bool, } +impl Clone for FloatHistogram { + fn clone(&self) -> Self { + #[cfg(test)] + copies::record_clone(); + Self { + reset_hint: self.reset_hint, + start_timestamp: self.start_timestamp, + schema: self.schema, + zero_threshold: self.zero_threshold, + zero_count: self.zero_count, + count: self.count, + sum: self.sum, + positive: self.positive.clone(), + negative: self.negative.clone(), + custom_values: self.custom_values.clone(), + layout: self.layout.clone(), + } + } +} + impl FloatHistogram { pub(crate) fn try_from_view(view: &HistogramView<'_>) -> Result { + #[cfg(test)] + copies::record_materialized(); Ok(Self { reset_hint: view.reset_hint, start_timestamp: view.start_timestamp, diff --git a/src/promql/src/histogram_eval/tests.rs b/src/promql/src/histogram_eval/tests.rs index 12e783b68f7..3c1834b4e3e 100644 --- a/src/promql/src/histogram_eval/tests.rs +++ b/src/promql/src/histogram_eval/tests.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; use std::collections::BTreeMap; use std::sync::Arc; @@ -22,16 +23,17 @@ use datatypes::series::histogram::{ use datatypes::series::samples::{FieldSpec, SampleRef, Series, SeriesSchema}; use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis}; +use super::float_histogram::copies; use super::float_histogram::{AddOutcome, FloatHistogram}; use super::{ HistogramAggregate, HistogramFunction, HistogramParameters, NativeHistogramFunction, aggregate_histograms, evaluate_histograms, evaluate_histograms_with_annotations, - histogram_quantiles, histogram_string, map_native_histogram, + histogram_quantiles, histogram_string, map_native_histogram, sum_or_avg, }; use crate::grid_series::merge_same_labelset; use crate::grid_series::reference::{ - GoldenCase, Labels, OwnedHistogram, Points, canonical_bits, close, golden_cases, labels_of, - metric_of, mixed_series, points_by_labels, points_of, + GoldenCase, Labels, OwnedHistogram, Points, assert_histogram_close, canonical_bits, close, + golden_cases, labels_of, metric_of, mixed_series, points_by_labels, points_of, }; fn schema(case: &GoldenCase) -> Arc { @@ -1065,3 +1067,81 @@ fn classic_histograms_keep_metadata_identity_until_after_evaluation() { ); } } + +/// A window of native histograms is folded by reference: the sum copies the accumulator it starts +/// from and nothing else, so four samples cost one deep copy rather than four. The fold itself is +/// the one the owned samples go through, bit for bit. +#[test] +fn a_histogram_sum_copies_one_sample_of_a_window() { + let window: Vec = [ + "0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///", + "0/0/0/4000000000000000/4008000000000000/0/1_1/4000000000000000///", + "0/0/0/4008000000000000/4010000000000000/0/1_1/4008000000000000///", + "0/0/0/4010000000000000/4014000000000000/0/1_1/4010000000000000///", + ] + .iter() + .map(|text| FloatHistogram::try_from_view(&OwnedHistogram::parse(text).view()).unwrap()) + .collect(); + + copies::take_clones(); + let (sum, reconciled) = sum_or_avg( + HistogramAggregate::Sum, + window.iter().map(|h| Ok(Cow::Borrowed(h))), + ) + .unwrap(); + assert_eq!( + copies::take_clones(), + 1, + "the sum copied every sample of the window, not only the accumulator it starts from" + ); + assert!(!reconciled); + + // The same window as owned samples, taken before the count: the accumulator arrives by value, + // so the fold copies nothing and must produce the result the borrowed fold produced. + let owned_samples: Vec> = + window.iter().cloned().map(Cow::Owned).collect(); + copies::take_clones(); + let (owned, _) = + sum_or_avg(HistogramAggregate::Sum, owned_samples.into_iter().map(Ok)).unwrap(); + assert_eq!(copies::take_clones(), 0, "an owned sample was copied"); + + let sum = sum.expect("the window sums to a histogram"); + assert_histogram_close( + &sum, + &owned.expect("the window sums to a histogram"), + 0.0, + "the borrowed sum", + ); +} + +/// The mean reads the window by reference too, and takes the same samples by value as the owned +/// fold does: the sum `kahan_add` keeps is copied either way, and the values agree bit for bit. +#[test] +fn a_histogram_mean_reads_its_window_by_reference() { + let window: Vec = [ + "0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///", + "0/0/0/4000000000000000/4008000000000000/0/1_1/4000000000000000///", + "0/0/0/4008000000000000/4010000000000000/0/1_1/4008000000000000///", + ] + .iter() + .map(|text| FloatHistogram::try_from_view(&OwnedHistogram::parse(text).view()).unwrap()) + .collect(); + + let borrowed = sum_or_avg( + HistogramAggregate::Avg, + window.iter().map(|h| Ok(Cow::Borrowed(h))), + ) + .unwrap(); + let owned = sum_or_avg( + HistogramAggregate::Avg, + window.iter().cloned().map(|h| Ok(Cow::Owned(h))), + ) + .unwrap(); + assert_eq!(borrowed.1, owned.1); + assert_histogram_close( + &borrowed.0.expect("the window averages to a histogram"), + &owned.0.expect("the window averages to a histogram"), + 0.0, + "the borrowed mean", + ); +} diff --git a/src/promql/src/instant_eval.rs b/src/promql/src/instant_eval.rs index 0b8b092bf4c..08e9ce51d3a 100644 --- a/src/promql/src/instant_eval.rs +++ b/src/promql/src/instant_eval.rs @@ -27,6 +27,22 @@ use crate::map::drop_metric_name; #[cfg(test)] mod tests; +/// The source rows [`select_instants`] examined, counted per row. +/// +/// The selector advances one cursor over the axis, so the rows it reads can never outnumber the +/// rows of the axis: one pass, not one per grid point. Counting them is what keeps a second pass +/// over the axis from being added back unobserved, since the selection it returns is the same +/// either way. +#[cfg(test)] +pub(crate) static AXIS_ROWS_READ: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +/// Counts one row of the axis examined; a no-op outside tests. +fn row_read() { + #[cfg(test)] + AXIS_ROWS_READ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); +} + /// Selects the one source row an anchored instant reads, with the lookback, offset and staleness /// rules of [`select_instants`]: the last sample at or before `at_ms` (shifted by `offset_ms`) and /// not more than the lookback before it, unless a stale marker is the latest sample there. @@ -101,10 +117,13 @@ pub(crate) fn instant_candidate_span( /// Selects source rows for an aligned PromQL instant-evaluation range. /// -/// Returns original logical row indices and evaluation timestamps from a complete source axis -/// of strictly increasing milliseconds. Any provided field's stale marker suppresses selection; +/// Returns original logical row indices and evaluation timestamps from a complete source axis of +/// strictly increasing milliseconds. Any provided field's stale marker suppresses selection; /// missing samples are not stale and remain selectable. /// +/// The ordering of the axis is [`RawTimestamps`]' own guarantee, checked when it was built, so this +/// walks the axis once -- one cursor advance per row -- and validates nothing about it again. +/// /// The two vectors it returns hold one `u64` source row index and one `i64` evaluation timestamp /// per selected point, so what it allocates is 16 bytes per point of the clipped grid rather than /// anything proportional to the samples of the axis. It reserves none of that: every caller grows @@ -178,6 +197,7 @@ pub fn select_instants( }; current_stale |= stale; } + row_read(); cursor += 1; } let Some(candidate) = cursor.checked_sub(1) else { diff --git a/src/promql/src/instant_eval/tests.rs b/src/promql/src/instant_eval/tests.rs index cd2b3926426..036c948a53a 100644 --- a/src/promql/src/instant_eval/tests.rs +++ b/src/promql/src/instant_eval/tests.rs @@ -15,6 +15,7 @@ use std::collections::BTreeMap; use std::ops::RangeInclusive; use std::sync::Arc; +use std::sync::atomic::Ordering; use datatypes::arrow::array::{Float64Array, UInt32Array}; use datatypes::arrow::buffer::ScalarBuffer; @@ -28,8 +29,8 @@ use datatypes::series::samples::{ use datatypes::series::time::{EvalGrid, GridSpec}; use super::{ - InstantCandidateSpan, info_sample_times, instant_candidate_span, select_instants, - timestamp_of_evaluated, timestamp_of_selection, + AXIS_ROWS_READ, InstantCandidateSpan, info_sample_times, instant_candidate_span, + select_instants, timestamp_of_evaluated, timestamp_of_selection, }; use crate::grid_series::GridSeriesBuilder; use crate::grid_series::reference::{Labels, labels_of, metric_of}; @@ -233,6 +234,42 @@ fn offset_extreme_timestamps_are_checked_without_large_fixture() { assert_eq!(times, [1_000_000]); } +/// A selection walks its axis once, whatever the grid it selects onto. +/// +/// The strictly increasing order of the timestamps is `RawTimestamps`' guarantee, checked when the +/// axis is built, so the selector has nothing left to validate: the rows it reads are the rows of +/// the axis, once each, and a second pass over them -- such as a re-check of their order -- would +/// show up here as twice as many. The selection itself is the same either way, which is why the +/// count has to be what the test reads. +#[test] +fn a_selection_reads_its_axis_once() { + let rows = 5_000; + let times: Vec = (0..rows).map(|row| row as i64).collect(); + let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0; rows]))); + let read = |range: RangeInclusive, step: i64| -> u64 { + AXIS_ROWS_READ.store(0, Ordering::Relaxed); + let (selected, _) = select(×, &fields, range, step, 10, 0).unwrap(); + assert!(!selected.is_empty(), "the selection is what reads the axis"); + AXIS_ROWS_READ.load(Ordering::Relaxed) + }; + // Every row is selected: one pass, no more. + assert_eq!( + read(0..=rows as i64 - 1, 1), + rows as u64, + "the axis was walked more than once" + ); + // Fewer points over the whole axis read it once as well: the pass follows the samples, not the + // points, so it never exceeds the rows of the axis however many points the grid holds. + for step in [10, 499] { + let visited = read(0..=rows as i64 - 1, step); + assert!( + visited <= rows as u64, + "step {step} read {visited} rows of a {} row axis", + rows + ); + } +} + #[test] fn instant_candidate_span_preserves_unaligned_and_wide_ranges() { // The query end is not on the start's lattice. The scan bound is 125, but stepping from 100 diff --git a/src/promql/src/native/canonicalize.rs b/src/promql/src/native/canonicalize.rs index 900b85d09c5..fefe2d88495 100644 --- a/src/promql/src/native/canonicalize.rs +++ b/src/promql/src/native/canonicalize.rs @@ -23,10 +23,10 @@ //! //! # What decides it //! -//! Only a rewrite can alias, and only one that writes a label its input already carries: -//! [`duplicates_canonical_labels`] is that fact, derived from the tree and the tags each source -//! reports. A tree with no such rewrite gains no merge at all, so a per-series query keeps every -//! scan partition and pays nothing for this pass. +//! Only a rewrite can alias, and only one that writes a label its input already carries or drops one +//! its output would still be told apart by: [`duplicates_canonical_labels`] is that fact, derived +//! from the tree and the tags each source reports. A tree with no such rewrite gains no merge at +//! all, so a per-series query keeps every scan partition and pays nothing for this pass. //! //! Where a merge is added is a property of the consumer, and of what the operations between the //! consumer and the rewrite do to one sample: @@ -45,6 +45,39 @@ //! A merge is never added below a raw scan, a scalar or aggregate state: it binds samples evaluated //! on a grid, and no consumer asks for one over those. //! +//! A node can be a source of duplicates without rewriting a label its input carries: a range +//! function that computes its result and `timestamp` in both of its forms drop the metadata labels - +//! the metric name, `__type__` and `__unit__` - and hand out a set those no longer tell apart. What +//! they produce is collected by their consumer - the root included - exactly as a rewrite of an +//! existing label is, unless +//! [`duplicates_canonical_labels`](crate::native::plan::duplicates_canonical_labels) proves that +//! nothing they drop was telling two of their series apart: a source bound to one metric names +//! every series it hands over, a source whose tags carry neither `__type__` nor `__unit__` holds no +//! series those two tell apart either, and the scan's key columns always decode a value. +//! `rate(some_metric[5m])` over such a scan is then a window that streams its scan, and the same +//! query over a selector that discovered several metrics keeps the collection it needs. The same +//! rule - the labels and the scan's keys - is what a math or histogram map, a parameterized map and +//! an arithmetic or `bool` scalar broadcast are judged by. +//! +//! The key half is not the tree's to answer, so the caller states it as `keys_survive`: two stored +//! series of one metric whose key column holds `NULL` in one and `''` in the other decode to an +//! absent and to an empty label, and the drop removes the empty one with the name, leaving both +//! under one label set. The rule is +//! [`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), +//! which the physical per-series scan reads to decide whether a node may stream and the planner +//! reads from the resolved table, so the two cannot disagree about a drop. A caller that cannot +//! state it passes `false`; the statements collected for that reason alone are counted +//! ([`leaves_a_drop_to_the_storage_keys`](crate::native::plan::leaves_a_drop_to_the_storage_keys)). +//! +//! A drop above a merge is not asked about the empty value it removes, which is what keeps a chain of +//! drops to one collection: [`NativeExpr::Merge`] hands over one series per canonical label set - the +//! comparison that treats an empty label value as no label - so no drop above it can fold two of its +//! series by dropping an empty one. The metadata labels a drop removes are another matter: a merge +//! keeps two series that differ in `__name__`, `__type__` or `__unit__`, so the labels half of the +//! proof is still asked above it. With that, `abs(abs(rate(x[5m])))` over a table whose key columns +//! can spell the `NULL`/`''` pair pays for the one merge at the bottom of the chain rather than one +//! per drop. +//! //! Canonicalizing a map's input is what makes the histogram case work: in //! `histogram_count(label_replace(x, "host", "same", "host", ".*"))` two scan partitions each //! rewrite one series to `{host="same"}` and `histogram_count` drops both - they are float samples - @@ -61,6 +94,8 @@ //! execution is the lowering's question. A merge that spans partitions uses a producer and the //! existing canonical router; downstream consumers read ordinary evaluated frames. +use std::sync::Arc; + use datafusion::error::Result as DfResult; use crate::native::plan::{ @@ -77,6 +112,20 @@ mod tests; /// source the caller says nothing about proves nothing, and a rewrite over it is collected. The /// index is what a leaf reads, so a tree with several sources is judged source by source. /// +/// `source_metrics` is the other half of that knowledge, indexed the same way: the one metric each +/// source is bound to, or `None` when it may hand over several. It is what lets a kernel that drops +/// the metadata labels keep every partition - see +/// [`duplicates_canonical_labels`](crate::native::plan::duplicates_canonical_labels) - and a caller +/// that cannot say collects instead. +/// +/// `keys_survive` is the last fact and the one the tree cannot carry: whether every scan below +/// `tree` still tells its series apart once a drop has normalized their labels +/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), +/// read from the resolved table). A key column that holds `NULL` in one series and `''` in another +/// decodes to an absent and to an empty label, and the empty one leaves with the metadata labels, so +/// `false` collects a drop the labels alone would have proved. A caller that cannot state it passes +/// `false`. +/// /// Every operation binding and every `source_index` stays where it was, so the rewrite changes which /// series a consumer reads and not what the query computes. The result is checked by /// [`NativeTree::try_new`], because a merge has to sit over an evaluated vector and the pass may @@ -84,10 +133,12 @@ mod tests; pub fn merge_label_rewrites( tree: &NativeTree, source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, ) -> DfResult { - let root = rewrite(tree.root(), source_tags)?; + let root = rewrite(tree.root(), source_tags, source_metrics, keys_survive)?; // The root is a consumer too: whatever reads this tree reads one series per label set. - NativeTree::try_new(canonical(root, source_tags)) + NativeTree::try_new(canonical(root, source_tags, source_metrics, keys_survive)) } /// Wraps `node` in a [`NativeExpr::Merge`] when its pieces can share a label set. @@ -100,14 +151,19 @@ pub fn merge_label_rewrites( /// ([`duplicates_canonical_labels`]), because collecting pieces that are already distinct is a /// barrier that buys nothing; /// - it is not already a merge, which would collect what is already collected. -fn canonical(node: NativeNode, source_tags: &[Vec]) -> NativeNode { +fn canonical( + node: NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> NativeNode { if !matches!(&node.output, NativeType::Samples(schema) if schema.eval_grid.is_some()) { return node; } if matches!(node.expr, NativeExpr::Merge { .. }) { return node; } - if !duplicates_canonical_labels(&node, source_tags) { + if !duplicates_canonical_labels(&node, source_tags, source_metrics, keys_survive) { return node; } // A merge reports its input's layout: the same fields on the same grid. @@ -122,7 +178,12 @@ fn canonical(node: NativeNode, source_tags: &[Vec]) -> NativeNode { /// Rewrites `node` bottom-up: its children first, then the merge its own consumer contract needs of /// them. -fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult { +fn rewrite( + node: &NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> DfResult { let expr = match &node.expr { // A source is the boundary it was bound to and there is nothing below it to rewrite. A // constant reads nothing. @@ -133,22 +194,27 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::Instant { - input: Box::new(rewrite(input, source_tags)?), + input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), op: op.clone(), }, NativeExpr::Timestamp { input } => NativeExpr::Timestamp { - input: Box::new(rewrite(input, source_tags)?), + input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), }, // A merge is the canonicalizer itself, so it asks nothing of its input: fragments are what it // was written to receive. NativeExpr::Merge { input } => NativeExpr::Merge { - input: Box::new(rewrite(input, source_tags)?), + input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), }, // A window folds a range of one series' samples and hands the answer over as that series, so // it reads the series whole. Over a raw scan the pieces are whole storage series already and // the merge above is a no-op; over an evaluated vector a rewrite can have aliased them. NativeExpr::Window { input, op } => NativeExpr::Window { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), }, // A map is point-wise, but a point can drop out of its output: the histogram functions drop @@ -157,7 +223,12 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::Map { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), }, // A parameterized map's parameters are scalars, rewritten but never collected - a scalar @@ -168,23 +239,38 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::ParameterizedMap { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: *op, parameters: parameters .iter() - .map(|parameter| rewrite(parameter, source_tags)) + .map(|parameter| rewrite(parameter, source_tags, source_metrics, keys_survive)) .collect::>>()?, }, // A fold answers its groups, so two pieces of one series in a group are summed as if the // query had written them twice: the merge is what makes one series arrive as one. NativeExpr::Aggregate { input, op } => NativeExpr::Aggregate { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), }, // Vector operations consume canonical series: ranking counts them, and presence checks and // folds must not silently consume overlapping fragments that a merge would reject. NativeExpr::Vector { input, op } => NativeExpr::Vector { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), }, // Each side of a binary or set operation is matched by label on its own, so each is @@ -195,13 +281,33 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::Binary { - left: Box::new(canonical(rewrite(left, source_tags)?, source_tags)), - right: Box::new(canonical(rewrite(right, source_tags)?, source_tags)), + left: Box::new(canonical( + rewrite(left, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), + right: Box::new(canonical( + rewrite(right, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), matching: matching.clone(), }, NativeExpr::Set { left, right, op } => NativeExpr::Set { - left: Box::new(canonical(rewrite(left, source_tags)?, source_tags)), - right: Box::new(canonical(rewrite(right, source_tags)?, source_tags)), + left: Box::new(canonical( + rewrite(left, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), + right: Box::new(canonical( + rewrite(right, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), }, // `info` pairs each base series with the info series that share its identifying labels, so it @@ -212,18 +318,33 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::Info { - base: Box::new(canonical(rewrite(base, source_tags)?, source_tags)), - info: Box::new(canonical(rewrite(info, source_tags)?, source_tags)), + base: Box::new(canonical( + rewrite(base, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), + info: Box::new(canonical( + rewrite(info, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), selector: selector.clone(), }, // `vector(s)` builds its series from a scalar, which carries no label set to collect. NativeExpr::VectorOf { input } => NativeExpr::VectorOf { - input: Box::new(rewrite(input, source_tags)?), + input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), }, // `scalar()` answers from the whole vector: a series that arrives in pieces would be seen // as several samples, so the pieces are collected where they still hold their samples. NativeExpr::ScalarOf { input } => NativeExpr::ScalarOf { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), }, // `topk` ranks the whole vector, so it counts canonical series; its parameter is a scalar, // rewritten but never collected - a scalar carries no label set. @@ -232,9 +353,19 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::ParameterizedTopK { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), - parameter: Box::new(rewrite(parameter, source_tags)?), + parameter: Box::new(rewrite( + parameter, + source_tags, + source_metrics, + keys_survive, + )?), }, // A histogram function folds the bucket series of one histogram, so overlapping fragments // must not reach it; its parameters are scalars and only need rewriting. @@ -243,11 +374,16 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult NativeExpr::ParameterizedHistogram { - input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), + input: Box::new(canonical( + rewrite(input, source_tags, source_metrics, keys_survive)?, + source_tags, + source_metrics, + keys_survive, + )), op: op.clone(), parameters: parameters .iter() - .map(|parameter| rewrite(parameter, source_tags)) + .map(|parameter| rewrite(parameter, source_tags, source_metrics, keys_survive)) .collect::>>()?, }, }; diff --git a/src/promql/src/native/canonicalize/tests.rs b/src/promql/src/native/canonicalize/tests.rs index 8b58bb8b21b..48ed4fda499 100644 --- a/src/promql/src/native/canonicalize/tests.rs +++ b/src/promql/src/native/canonicalize/tests.rs @@ -252,6 +252,31 @@ fn set_over(left: NativeNode, right: NativeNode, out: Arc) -> NativeNo } } +/// `last_over_time(inner[5m])`, a window that folds `inner` onto `out` and keeps the metric name. +fn keeping_window_over( + inner: NativeNode, + input_grid: Option>, + out: Arc, +) -> NativeNode { + NativeNode { + expr: NativeExpr::Window { + input: Box::new(inner), + op: BoundWindow { + function: RangeFunction::LastOverTime, + window: RangeWindow { + range_ms: 300_000, + offset_ms: 0, + anchor_ms: None, + modifier: None, + }, + input_grid, + output_grid: Arc::clone(&out), + }, + }, + output: evaluated(fields(1), out), + } +} + /// `rate(inner[5m])`, a window that folds `inner` onto `out` and drops the metric name. fn window_over( inner: NativeNode, @@ -328,9 +353,54 @@ fn merges(tree: &NativeTree) -> usize { } /// Runs the pass over `root`, whose sources report `source_tags`. +/// +/// Nothing is stated about which metric each source reads, so a kernel that drops the metadata +/// labels over one of them proves nothing and is collected. fn passed(root: NativeNode, source_tags: &[Vec]) -> NativeTree { + passed_over(root, source_tags, &[]) +} + +/// Runs the pass over `root`, whose sources report `source_tags` and read the metric each of +/// `source_metrics` names, indexed the way `merge_label_rewrites` takes it. +/// +/// The scans below these fixtures key their series by columns that always decode a value, which is +/// the case a metadata drop has to be proved against the labels alone: `NULL` and `''` in one key +/// column are what [`passed_over_nullable_keys`] states instead. +fn passed_over( + root: NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], +) -> NativeTree { + passed_over_with_keys(root, source_tags, source_metrics, true) +} + +/// The same pass over a tree whose scans key their series by a column that can hold `NULL` in one +/// series and `''` in another, so no metadata drop below it can be proved by the labels alone. +fn passed_over_nullable_keys( + root: NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], +) -> NativeTree { + passed_over_with_keys(root, source_tags, source_metrics, false) +} + +/// Runs the pass over `root` with the key fact the caller states, which is what +/// `merge_label_rewrites` takes as `keys_survive`. +fn passed_over_with_keys( + root: NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> NativeTree { let tree = NativeTree::try_new(root).expect("the fixture is a legal tree"); - merge_label_rewrites(&tree, source_tags).expect("the pass produces a legal tree") + merge_label_rewrites(&tree, source_tags, source_metrics, keys_survive) + .expect("the pass produces a legal tree") +} + +/// The labels a scan bound to one metric reports: its tag, and the name the binding gives every +/// series it hands over, which is not a tag column. +fn named_scan_tags() -> Vec> { + vec![tags(&["host", "__name__"])] } fn tags(names: &[&str]) -> Vec { @@ -438,7 +508,7 @@ fn presence_and_folding_vector_operations_collect_aliasing_inputs() { }; assert!(matches!(input.expr, NativeExpr::Map { .. }), "{op:?}"); - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); assert_eq!(1, merges(&twice), "{op:?}"); assert_eq!(tree.to_string(), twice.to_string(), "{op:?}"); } @@ -493,7 +563,7 @@ fn rewrites_after_ranking_or_grouping_are_collected() { ), "{name}" ); - let twice = merge_label_rewrites(&tree, &source_tags).unwrap(); + let twice = merge_label_rewrites(&tree, &source_tags, &[], true).unwrap(); assert_eq!(tree.to_string(), twice.to_string(), "{name}"); assert_eq!(1, merges(&twice), "{name}"); } @@ -504,7 +574,8 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() { let out = grid(0, 1000, 2); // A match answers one series per label set, but those labels are its operands': renaming `host` // to a constant above it puts both sides' results under one set, so the consumer above the - // rewrite is the one that has to collect them. The match itself reads its operands as they are. + // rewrite is the one that has to collect them. The match itself reads its operands as they are; + // the one on the right drops the metadata labels, so its own answer is collected. let matched = binary_over(selected(&out), abs_map(selected(&out)), Arc::clone(&out)); let renamed = map_ops( matched, @@ -513,7 +584,11 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() { ))], ); let tree = passed(renamed, &scan_tags()); - assert_eq!(1, merges(&tree)); + assert_eq!( + 2, + merges(&tree), + "the rewrite's own collection, and the one the name-dropping operand hands out: {tree}" + ); let NativeExpr::Merge { input } = &tree.root().expr else { panic!("the statement must collect the rewrite above the match"); }; @@ -524,9 +599,9 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() { matches!(input.expr, NativeExpr::Binary { .. }), "the match is not handed the rewritten operand" ); - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); assert_eq!(tree.to_string(), twice.to_string()); - assert_eq!(1, merges(&twice)); + assert_eq!(2, merges(&twice)); } #[test] @@ -586,18 +661,21 @@ fn a_pointwise_map_reads_each_piece_as_it_is() { abs_map(rewrite(selected(&out), "host", "host")), &scan_tags(), ); - assert_eq!(1, merges(&tree)); - // The map stays the root and its input is the merge: a map reads pieces point-wise, but a point - // it reads can be one it drops, so the collection has to happen below it. - let NativeExpr::Map { input, .. } = &tree.root().expr else { + // Two collections: one below the map, because a map reads pieces point-wise and a point it + // reads can be one it drops, and one above it, for the metadata labels the map itself drops. + assert_eq!(2, merges(&tree), "{tree}"); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the map hands out: {tree}"); + }; + let NativeExpr::Map { input, .. } = &input.expr else { panic!("the root stays the point-wise map"); }; let NativeExpr::Merge { input } = &input.expr else { panic!("the map is handed one series per label set"); }; assert!(matches!(input.expr, NativeExpr::Map { .. })); - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); - assert_eq!(1, merges(&twice)); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + assert_eq!(2, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } @@ -613,8 +691,15 @@ fn a_sample_dropping_map_collects_below_itself() { ))], ); let tree = passed(histogram_count(aliasing), &scan_tags()); - assert_eq!(1, merges(&tree), "one merge, below the sample-dropping map"); - let NativeExpr::Map { input, .. } = &tree.root().expr else { + assert_eq!( + 2, + merges(&tree), + "one merge below the sample-dropping map, one for the labels it drops" + ); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what `histogram_count` hands out: {tree}"); + }; + let NativeExpr::Map { input, .. } = &input.expr else { panic!("the root stays `histogram_count`"); }; let NativeExpr::Merge { input } = &input.expr else { @@ -633,36 +718,51 @@ fn a_sample_dropping_map_collects_below_itself() { ); // A second pass over the collected tree changes nothing: the merge answers one series per label // set, so nothing below the map aliases any more. - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); - assert_eq!(1, merges(&twice)); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + assert_eq!(2, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } #[test] -fn a_pointwise_map_over_a_clean_input_gains_no_merge() { +fn a_pointwise_map_collects_its_own_name_drop() { let out = grid(0, 1000, 2); - // No rewrite anywhere in the tree, so no two pieces can share a label set. + // No rewrite anywhere in the tree, so no two pieces were told apart by a label one of them + // wrote - but `abs` drops the metadata labels, and the drop is a source of duplicates of its + // own: a key column that holds `NULL` in one stored series and `''` in another decodes to an + // absent and to an empty label, and both are gone once the empty one goes. Each map's answer is + // therefore collected, and both sit above the map that hands them out. let tree = passed(abs_map(abs_map(selected(&out))), &scan_tags()); - assert_eq!(0, merges(&tree)); - let NativeExpr::Map { input, .. } = &tree.root().expr else { - panic!("the root stays the point-wise map"); + assert_eq!(2, merges(&tree), "{tree}"); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the outer map hands out: {tree}"); }; - assert!(matches!(input.expr, NativeExpr::Map { .. })); - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + let NativeExpr::Map { input, .. } = &input.expr else { + panic!("the merge sits above the map, not below it: {tree}"); + }; + let NativeExpr::Merge { input } = &input.expr else { + panic!("the outer map reads one series per label set: {tree}"); + }; + assert!(matches!(input.expr, NativeExpr::Map { .. }), "{tree}"); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + assert_eq!(2, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } #[test] -fn a_sample_dropping_map_over_a_fresh_destination_gains_no_merge() { +fn a_sample_dropping_map_over_a_fresh_destination_collects_its_own_drop() { let out = grid(0, 1000, 2); // `dst` names no label the scan reports, so the rewrite cannot alias and the map's input needs - // no collection even though the map drops samples. + // no collection even though the map drops samples - but the metadata labels `histogram_count` + // drops are a source of duplicates of their own, so its answer is collected. let tree = passed( histogram_count(rewrite(selected(&out), "dst", "host")), &scan_tags(), ); - assert_eq!(0, merges(&tree), "a fresh destination cannot alias"); - let NativeExpr::Map { input, .. } = &tree.root().expr else { + assert_eq!(1, merges(&tree), "a fresh destination cannot alias: {tree}"); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what `histogram_count` hands out: {tree}"); + }; + let NativeExpr::Map { input, .. } = &input.expr else { panic!("the root stays `histogram_count`"); }; assert!(matches!(input.expr, NativeExpr::Map { .. })); @@ -680,10 +780,15 @@ fn a_parameterized_map_collects_below_itself() { ), &scan_tags(), ); - assert_eq!(1, merges(&tree)); + // One below the map, for the point a clamp can empty, and one above it, for the metadata labels + // the kernel drops. + assert_eq!(2, merges(&tree), "{tree}"); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the clamp hands out: {tree}"); + }; let NativeExpr::ParameterizedMap { input, parameters, .. - } = &tree.root().expr + } = &input.expr else { panic!("the root stays the parameterized map"); }; @@ -696,8 +801,8 @@ fn a_parameterized_map_collects_below_itself() { matches!(parameters[0].expr, NativeExpr::Scalar(_)), "a scalar argument is not collected" ); - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); - assert_eq!(1, merges(&twice)); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + assert_eq!(2, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } @@ -709,12 +814,16 @@ fn each_side_of_a_match_is_collected_on_its_own() { binary_over(aliasing(), abs_map(selected(&out)), Arc::clone(&out)), &scan_tags(), ); - assert_eq!(1, merges(&tree), "only the aliasing side needs collecting"); + assert_eq!( + 2, + merges(&tree), + "the aliasing side and the name-dropping side are each collected: {tree}" + ); let NativeExpr::Binary { left, right, .. } = &tree.root().expr else { panic!("the root stays the match"); }; assert!(matches!(left.expr, NativeExpr::Merge { .. })); - assert!(!matches!(right.expr, NativeExpr::Merge { .. })); + assert!(matches!(right.expr, NativeExpr::Merge { .. })); } #[test] @@ -736,9 +845,10 @@ fn a_set_operation_collects_each_side() { #[test] fn a_window_collects_an_aliasing_child() { let out = grid(0, 1000, 2); - // A window over an evaluated vector folds one series' range, so it reads the series whole. + // A window over an evaluated vector folds one series' range, so it reads the series whole. The + // window keeps the metric name here, so the rewrite below it is the only source of duplicates. let tree = passed( - window_over( + keeping_window_over( rewrite(selected(&out), "host", "host"), Some(Arc::clone(&out)), Arc::clone(&out), @@ -756,10 +866,614 @@ fn a_window_collects_an_aliasing_child() { fn a_window_over_a_raw_scan_collects_nothing() { let out = grid(0, 1000, 2); // The scan hands over whole storage series, and a merge cannot take raw samples anyway. - let tree = passed(window_over(raw_input(0), None, out), &scan_tags()); + let tree = passed(keeping_window_over(raw_input(0), None, out), &scan_tags()); assert_eq!(0, merges(&tree)); } +#[test] +fn a_name_dropping_window_is_collected_by_its_consumer() { + let out = grid(0, 1000, 2); + // `rate` drops the metric name, which is part of the identity a merge groups by: a caller that + // says nothing about which metric the scan reads collects what the window hands out. + let tree = passed( + window_over(raw_input(0), None, Arc::clone(&out)), + &named_scan_tags(), + ); + assert_eq!(1, merges(&tree)); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the window hands out: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Window { .. }), + "the merge sits above the window, not below it" + ); +} + +#[test] +fn a_name_dropping_window_over_one_metric_streams() { + let out = grid(0, 1000, 2); + // The scan is bound to one metric, so every series it hands over is named the same thing: + // dropping that name tells none of them apart, and the window keeps every partition. + // + // The key columns are the other half of that proof, and this fixture states the half that holds: + // every one of them always decodes a value, so the empty label value the drop removes with the + // name never stood for a second stored series. + let tree = passed_over( + window_over(raw_input(0), None, Arc::clone(&out)), + &named_scan_tags(), + &[Some(Arc::from("some_metric"))], + ); + assert_eq!(0, merges(&tree), "{tree}"); + assert!( + matches!(tree.root().expr, NativeExpr::Window { .. }), + "the root stays the window: {tree}" + ); +} + +/// Two stored series of one metric that differ only in a tag column holding `NULL` in one and `''` +/// in the other: `{host="web-1", zone=""}` and `{host="web-1"}` are two series to the scan, and +/// `rate` drops the metric name and the empty `zone` with it, so both leave the window as +/// `{host="web-1"}`. +/// +/// The labels alone cannot tell them apart - they share host, zone is about to go, and the name is +/// the one every series of the scan shares - so the scan's key columns are what decides the answer. +/// A key column that can hold that pair is the fact that says so, and the drop is collected: the +/// merge is where the second value of one label set at one position is refused, which is +/// Prometheus's answer to two series of one label set rather than two pieces of one series. +/// +/// The execution half of this - that the merge refuses the pair instead of handing both over - is +/// `two_series_onto_one_label_set_after_a_window_are_rejected` in `native::execute::tests`. +#[test] +fn a_nullable_key_tag_collects_the_window_that_drops_an_empty_value() { + let out = grid(0, 1000, 2); + // A scan of one metric whose tags are `host` and `zone`, the second of which is a nullable key + // column: the pair above is one storage key away from being two. + let scan = [tags(&["host", "zone", "__name__"])]; + let single = [Some(Arc::::from("some_metric"))]; + let tree = passed_over_nullable_keys( + window_over(raw_input(0), None, Arc::clone(&out)), + &scan, + &single, + ); + assert!( + merges(&tree) >= 1, + "a nullable key column keeps the collection `rate` needs: {tree}" + ); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the window hands out: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Window { .. }), + "the merge sits above the window: {tree}" + ); + + // The same tree over a scan whose keys always decode a value: the labels prove the drop, and + // the window keeps every partition. + let tree = passed_over( + window_over(raw_input(0), None, Arc::clone(&out)), + &scan, + &single, + ); + assert_eq!( + 0, + merges(&tree), + "non-nullable keys keep the proof, so nothing is collected: {tree}" + ); +} + +#[test] +fn a_name_dropping_window_over_several_metrics_is_collected() { + let out = grid(0, 1000, 2); + // A selector that discovered several metrics binds no single name, so two of its series can be + // told apart by the name alone - and dropping it folds them into one label set. + let tree = passed_over( + window_over(raw_input(0), None, Arc::clone(&out)), + &named_scan_tags(), + &[None], + ); + assert_eq!(1, merges(&tree), "{tree}"); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "the root collects what the window hands out: {tree}" + ); +} + +#[test] +fn a_metadata_tag_in_the_binding_fails_closed() { + let out = grid(0, 1000, 2); + // `__type__` and `__unit__` go with the name, and a scan that carries one of them as a tag can + // hold two series that differ only there. A fixed metric name proves nothing about them, so the + // drop stays collected. + for metadata in ["__type__", "__unit__"] { + let scan = vec![tags(&["host", "__name__", metadata])]; + let tree = passed_over( + window_over(raw_input(0), None, Arc::clone(&out)), + &scan, + &[Some(Arc::from("some_metric"))], + ); + assert_eq!(1, merges(&tree), "{metadata}: {tree}"); + } +} + +/// The name a name-dropping kernel removes is the one every series of a single-metric scan shares, +/// and nothing else it drops tells two of them apart, so a math map, a histogram map, a +/// parameterized math kernel and an arithmetic scalar broadcast handed such a source keep every +/// partition. +/// +/// The scan's key columns are the other half of that proof, and this is the case they hold for: +/// every one of them always decodes a value, so the empty label value the drop removes with the name +/// never stood for a second stored series. [`a_name_dropping_kernel_over_a_nullable_key_is_collected`] +/// is the same fixtures over a scan whose keys can spell that pair. +#[test] +fn a_name_dropping_kernel_over_one_metric_keeps_its_partitions() { + let out = grid(0, 1000, 2); + let single = [Some(Arc::::from("some_metric"))]; + let kept = |name: &str, root: NativeNode| { + let tree = passed_over(root, &named_scan_tags(), &single); + assert_eq!(0, merges(&tree), "`{name}` cannot alias: {tree}"); + assert!( + !matches!(tree.root().expr, NativeExpr::Merge { .. }), + "`{name}` hands over one series per label set already: {tree}" + ); + }; + // A math map. + kept("abs", abs_map(selected(&out))); + // A histogram map. + kept("histogram_count", histogram_count(selected(&out))); + // A parameterized math kernel. + kept( + "clamp_min", + clamp_min(selected(&out), scalar(0.0, Arc::clone(&out))), + ); + // An arithmetic scalar broadcast, `x * 2`. + kept( + "x * 2", + binary_over( + selected(&out), + scalar(2.0, Arc::clone(&out)), + Arc::clone(&out), + ), + ); +} + +/// The same kernels over a binding that carries `__type__` or `__unit__` as a tag are collected at +/// the root. +/// +/// Those two go with the name when a kernel drops it, so two series that differ only there meet once +/// the drop removes them, whatever the scan's key columns read otherwise: this is the labels half +/// failing, not the key half. It is asked before the key columns are. +#[test] +fn a_name_dropping_kernel_over_a_metadata_tag_is_collected() { + let out = grid(0, 1000, 2); + let single = [Some(Arc::::from("some_metric"))]; + let collected = |name: &str, tags: &[Vec], root: NativeNode| { + let tree = passed_over(root, tags, &single); + assert_eq!(1, merges(&tree), "`{name}` drops the name: {tree}"); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "`{name}` hands out one series per label set: {tree}" + ); + }; + for metadata in ["__type__", "__unit__"] { + let scan = vec![tags(&["host", "__name__", metadata])]; + collected("abs", &scan, abs_map(selected(&out))); + collected("histogram_count", &scan, histogram_count(selected(&out))); + collected( + "clamp_min", + &scan, + clamp_min(selected(&out), scalar(0.0, Arc::clone(&out))), + ); + collected( + "x * 2", + &scan, + binary_over( + selected(&out), + scalar(2.0, Arc::clone(&out)), + Arc::clone(&out), + ), + ); + } +} + +/// A name-dropping kernel above another one asks for no collection of its own, and neither does a +/// broadcast above a window that drops the name: `rate(x[5m]) * 2` plans one window and one +/// broadcast, and `abs(abs(rate(x[5m])))` one window and two maps, with no merge in either. +/// +/// That is the answer over a scan whose key columns always decode a value. Over one that can hold +/// `NULL` in one series and `''` in the other, the drop the labels cannot prove is collected - but +/// only once, at the bottom of the chain: the merge answers the empty-value half of every drop above +/// it, because it hands over one series per label set under the comparison that treats an empty +/// value as no label. Each extra merge would buffer its whole input to collect a pair the drop above +/// it cannot fold. The labels half is a different matter and is still asked - see +/// [`a_merge_below_a_drop_answers_only_the_empty_values`]. +#[test] +fn a_drop_above_a_name_dropping_kernel_needs_no_collection() { + let out = grid(0, 1000, 2); + // What the planner states for a scan of one metric: its tag columns and the name the binding + // gives every series it hands over. + let scan = [tags(&["host", "zone", "__name__"])]; + let single = [Some(Arc::::from("some_metric"))]; + let rate = || window_over(raw_input(0), None, Arc::clone(&out)); + // `rate(x[5m]) * 2`: the broadcast drops the metadata labels the window dropped already. + let broadcast = binary_over(rate(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)); + // `abs(abs(rate(x[5m])))`: two math kernels above the window, neither of which can fold a pair + // the window left distinct. + let nested = abs_map(abs_map(rate())); + for (query, root) in [ + ("rate(x[5m]) * 2", broadcast), + ("abs(abs(rate(x[5m])))", nested), + ] { + let tree = passed_over(root, &scan, &single); + assert_eq!(0, merges(&tree), "`{query}` plans a merge: {tree}"); + } +} + +/// The same two queries over a scan whose keys can spell one series as `NULL` and another as `''`: +/// the drop is collected once, at the bottom of the chain, and the drops above it cost nothing. +/// +/// One merge is what a nullable key column costs a statement, whatever the length of the chain above +/// it. The merge sits above the window - the bottom-most drop - so every map and broadcast above it +/// is asked nothing: what they read is already one series per canonical label set. +#[test] +fn a_nullable_key_column_collects_a_chain_of_drops_once() { + let out = grid(0, 1000, 2); + // The same facts as the test above, with a key column that can hold `NULL` in one series and + // `''` in the other, which no labels can prove apart. + let scan = [tags(&["host", "zone", "__name__"])]; + let single = [Some(Arc::::from("some_metric"))]; + let rate = || window_over(raw_input(0), None, Arc::clone(&out)); + let broadcast = binary_over(rate(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)); + let nested = abs_map(abs_map(rate())); + for (query, root) in [ + ("rate(x[5m]) * 2", broadcast), + ("abs(abs(rate(x[5m])))", nested), + ] { + let tree = passed_over_nullable_keys(root, &scan, &single); + assert_eq!( + 1, + merges(&tree), + "`{query}` pays for the one merge its key columns need, not one per drop: {tree}" + ); + } +} + +/// A nullable key column collects the drop it is about, and that merge sits above the drop rather +/// than anywhere else in the chain: the window's answer is what the two stored series arrive at. +#[test] +fn a_nullable_key_column_collects_the_bottom_most_drop() { + let out = grid(0, 1000, 2); + let scan = [tags(&["host", "zone", "__name__"])]; + let single = [Some(Arc::::from("some_metric"))]; + // `abs(abs(rate(x[5m])))` over the nullable key: the merge is the window's consumer, and both + // maps above it read what it hands over. + let tree = passed_over_nullable_keys( + abs_map(abs_map(window_over(raw_input(0), None, Arc::clone(&out)))), + &scan, + &single, + ); + assert_eq!(1, merges(&tree), "{tree}"); + let NativeExpr::Map { input, .. } = &tree.root().expr else { + panic!("the root stays the outer map: {tree}"); + }; + let NativeExpr::Map { input, .. } = &input.expr else { + panic!("the two maps are not collected apart: {tree}"); + }; + let NativeExpr::Merge { input } = &input.expr else { + panic!("the merge sits above the window: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Window { .. }), + "the merge sits above the window, not below it: {tree}" + ); +} + +/// A merge below a drop answers the empty-value half of the proof only: a merge keeps two series +/// that differ in `__name__`, `__type__` or `__unit__`, and removing those is exactly what a +/// metadata drop does. Two metrics whose ordinary labels are equal are therefore still collected +/// above a merge, one merge per collision. +/// +/// A selector that discovered several metrics is that shape: each metric's window is collected for +/// the name it drops, and the map above the merge is collected again for the name the merge kept. +#[test] +fn a_merge_below_a_drop_answers_only_the_empty_values() { + let out = grid(0, 1000, 2); + let scan = [tags(&["host", "__name__"])]; + // The source may hand over several metrics, which is what leaves the labels half unproved above + // the window's merge. + let tree = passed_over( + abs_map(window_over(raw_input(0), None, Arc::clone(&out))), + &scan, + &[None], + ); + assert_eq!( + 2, + merges(&tree), + "the window's name drop and the map's are two collisions, and the merge between them \ + cannot fold the names they remove: {tree}" + ); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the map hands out: {tree}"); + }; + let NativeExpr::Map { input, .. } = &input.expr else { + panic!("the root stays the map: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Merge { .. }), + "the window's answer is collected below the map: {tree}" + ); +} + +/// A `Merge` hands over one series per canonical label set - the one that treats an empty label +/// value as no label - so no drop above it can fold two of its series by removing an empty value. +/// The kernels above a merge therefore need no collection of their own for the empty-value half, one +/// merge per collision rather than one per kernel that repeats the collapse; the labels half is a +/// different matter (`a_merge_below_a_drop_answers_only_the_empty_values`). +#[test] +fn a_drop_above_a_merge_needs_no_collection_of_its_own() { + let out = grid(0, 1000, 2); + let single = [Some(Arc::::from("some_metric"))]; + // `label_replace(x, "host", "$1", "host", "(.*)")`: every host is renamed to one value, so the + // rewrite folds the scan's series into one label set and the pass collects it there. + let aliasing = || rewrite(selected(&out), "host", "host"); + + // `label_replace(x, "host", "$1", "host", "(.*)") * 2`: the broadcast drops the metadata labels + // above a merge, which cannot fold two series the merge already told apart. + let tree = passed_over( + binary_over(aliasing(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)), + &named_scan_tags(), + &single, + ); + assert_eq!(1, merges(&tree), "`label_replace(...) * 2`: {tree}"); + let NativeExpr::Binary { left, .. } = &tree.root().expr else { + panic!("the root stays the broadcast: {tree}"); + }; + let NativeExpr::Merge { input } = &left.expr else { + panic!("the vector side is handed one series per label set: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Map { .. }), + "the merge sits above the rewrite, and no second one above the match: {tree}" + ); + + // `abs(abs(rate(label_replace(...)[5m])))`: the rewrite's collision is the one collection, and + // neither map above it asks for another - their own name drop is over series a merge holds. + let tree = passed_over( + abs_map(abs_map(window_over( + aliasing(), + Some(Arc::clone(&out)), + Arc::clone(&out), + ))), + &named_scan_tags(), + &single, + ); + assert_eq!( + 1, + merges(&tree), + "`abs(abs(rate(label_replace(...)[5m])))`: {tree}" + ); + let NativeExpr::Map { input, .. } = &tree.root().expr else { + panic!("the root stays the outer map: {tree}"); + }; + let NativeExpr::Map { input, .. } = &input.expr else { + panic!("the two maps are not collected apart: {tree}"); + }; + let NativeExpr::Window { input, .. } = &input.expr else { + panic!("the inner map reads the window: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Merge { .. }), + "the one merge sits above the rewrite: {tree}" + ); + // A second pass changes nothing: the merge answers one series per label set, and nothing above + // it collects again. + let twice = merge_label_rewrites(&tree, &named_scan_tags(), &single, true).unwrap(); + assert_eq!(1, merges(&twice), "{twice}"); + assert_eq!(tree.to_string(), twice.to_string()); +} + +/// A collection below a merge still happens: two series the tree folds into one label set there are +/// collected by the merge the pass puts above the rewrite, and the drop above them asks for nothing +/// more - the merge is what makes the two arrive as one. +#[test] +fn a_collision_below_a_merge_is_collected_there() { + let out = grid(0, 1000, 2); + let single = [Some(Arc::::from("some_metric"))]; + // `rate(label_replace(x, "host", "$1", "host", "(.*)")[5m])`: every host is renamed to one + // value, so the rewrite folds the scan's series into one label set - below the window, whose + // `rate` drops the metric name on top of that. + let tree = passed_over( + window_over( + rewrite(selected(&out), "host", "host"), + Some(Arc::clone(&out)), + Arc::clone(&out), + ), + &named_scan_tags(), + &single, + ); + assert_eq!( + 1, + merges(&tree), + "the rewrite's own collection is what collects the collision: {tree}" + ); + let NativeExpr::Window { input, .. } = &tree.root().expr else { + panic!("the root stays the window: {tree}"); + }; + let NativeExpr::Merge { input } = &input.expr else { + panic!("the window reads the collected rewrite: {tree}"); + }; + assert!( + matches!(input.expr, NativeExpr::Map { .. }), + "the merge sits above the rewrite, not above the window: {tree}" + ); + let twice = merge_label_rewrites(&tree, &named_scan_tags(), &single, true).unwrap(); + assert_eq!(1, merges(&twice), "{twice}"); + assert_eq!(tree.to_string(), twice.to_string()); +} + +/// A name write is collected even when a merge already holds the series below it: rewriting a name +/// can still put two metrics whose ordinary labels are equal under one identity, which is the rule +/// the kernels' name-drop proof states. +/// +/// `abs(label_replace(label_replace(x, "__name__", "$1", "host", "(.*)"), "host", "", "", ""))` +/// deletes the host the name was derived from, so the name is the only label still telling two +/// series apart. `single_metric_name` in `native::plan` is what refuses to carry a written name: it +/// would otherwise hand the scan's own name back, and a binding whose key columns always decode a +/// value would then make the drop look injective. +#[test] +fn a_written_name_is_not_the_name_the_scan_binds() { + let out = grid(0, 1000, 2); + // The scan reads one metric, so nothing but the written name is left to tell two of its series + // apart. + let single = [Some(Arc::::from("some_metric"))]; + let rewritten = map_ops( + map_ops( + selected(&out), + vec![MapOp::Label(Arc::new( + LabelRewrite::replace("__name__", "$1", "host", "(.*)").unwrap(), + ))], + ), + vec![MapOp::Label(Arc::new( + LabelRewrite::replace("host", "", "", "").unwrap(), + ))], + ); + let tree = passed_over(abs_map(rewritten), &named_scan_tags(), &single); + // The rewrite of the name and the rewrite of the `host` it was derived from are collected in + // their own right, and the root merge is the `abs` above them - the node whose answer the + // destination check decides. + assert_eq!(3, merges(&tree), "{tree}"); + let NativeExpr::Merge { input: abs_node } = &tree.root().expr else { + panic!("the root collects what `abs` hands out: {tree}"); + }; + let NativeExpr::Map { input: below, op } = &abs_node.expr else { + panic!("the collected node is the map above the rewrites: {tree}"); + }; + assert!( + op.ops.iter().any(|op| matches!(op, MapOp::Math(_))), + "the root merge sits above `abs`, not above the rewrites: {tree}" + ); + let NativeExpr::Merge { input: delete } = &below.expr else { + panic!("`abs` reads the collected rewrite: {tree}"); + }; + let NativeExpr::Map { + input: name_write, .. + } = &delete.expr + else { + panic!("the merge below `abs` sits above the delete: {tree}"); + }; + assert!( + matches!(name_write.expr, NativeExpr::Merge { .. }), + "the name write is collected below the delete: {tree}" + ); +} + +#[test] +fn a_timestamp_is_collected_by_its_consumer() { + let out = grid(0, 1000, 2); + // `timestamp` reports a sample's own time under the metric name dropped, so a caller that says + // nothing about the scan's metric collects its answer at the root as well. + let stamped = || NativeNode { + expr: NativeExpr::Timestamp { + input: Box::new(selected(&out)), + }, + output: evaluated(fields(1), Arc::clone(&out)), + }; + let tree = passed(stamped(), &named_scan_tags()); + assert_eq!(1, merges(&tree)); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "the root collects what the timestamp hands out: {tree}" + ); + + // The scan reads one metric, so the name it drops told none of its series apart: the timestamp + // keeps every partition. + let tree = passed_over( + stamped(), + &named_scan_tags(), + &[Some(Arc::from("some_metric"))], + ); + assert_eq!(0, merges(&tree), "{tree}"); + assert!( + matches!(tree.root().expr, NativeExpr::Timestamp { .. }), + "the root stays the timestamp: {tree}" + ); +} + +/// The three nodes that drop the metadata labels on their own are judged by the labels: one metric +/// name over every series, and no metadata tag among them. +/// +/// A name-dropping window, `timestamp(x)` and a `timestamp` over an evaluated input all drop the +/// metadata labels. A source bound to one metric names every series it hands over, so that name told +/// none of them apart and the drop is proved; a source that may hand over several is collected, and +/// so is one that carries `__type__` or `__unit__` among its tags. A window that keeps the name +/// drops nothing and hands its input's identity on. +#[test] +fn a_name_dropping_node_is_proved_by_the_labels_alone() { + let out = grid(0, 1000, 2); + let named = named_scan_tags(); + let single = [Some(Arc::::from("some_metric"))]; + let collected = + |name: &str, tags: &[Vec], metrics: &[Option>], root: NativeNode| { + let tree = passed_over(root, tags, metrics); + assert_eq!( + 1, + merges(&tree), + "`{name}` drops the metadata labels: {tree}" + ); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "the root collects `{name}`: {tree}" + ); + }; + let kept = |name: &str, root: NativeNode| { + let tree = passed_over(root, &named, &single); + assert_eq!(0, merges(&tree), "`{name}` cannot alias: {tree}"); + assert!( + !matches!(tree.root().expr, NativeExpr::Merge { .. }), + "the root stays `{name}`: {tree}" + ); + }; + // `timestamp(x)` over the scan, `rate(x[5m])`, and `timestamp(rate(x[5m]))` over an evaluated + // input: the three nodes that drop the metadata labels themselves. + let stamps = || selection(raw_input(0), Arc::clone(&out), InstantValue::Timestamp); + let drops = || window_over(raw_input(0), None, Arc::clone(&out)); + let evaluated_stamps = || NativeNode { + expr: NativeExpr::Timestamp { + input: Box::new(drops()), + }, + output: evaluated(fields(1), Arc::clone(&out)), + }; + + // The labels hold: one metric names every series, and no metadata tag is among the tags. + kept("timestamp", stamps()); + kept("rate", drops()); + kept("timestamp(rate)", evaluated_stamps()); + // A source that may hand over several metrics is not proved: the name it drops is what tells + // two of them apart. + collected("timestamp", &named, &[None], stamps()); + collected("rate", &named, &[None], drops()); + collected("timestamp(rate)", &named, &[None], evaluated_stamps()); + // A binding that keys a series by a metadata label is collected: that label goes with the name, + // so two series that differ only there meet. + for metadata in ["__type__", "__unit__"] { + let mut typed = named.clone(); + typed[0].push(metadata.to_string()); + collected("timestamp", &typed, &single, stamps()); + collected("rate", &typed, &single, drops()); + collected("timestamp(rate)", &typed, &single, evaluated_stamps()); + } + // A window that keeps the name drops nothing, so nothing is asked of it. + let tree = passed_over( + keeping_window_over(raw_input(0), None, Arc::clone(&out)), + &named, + &single, + ); + assert_eq!(0, merges(&tree), "`last_over_time` keeps the name: {tree}"); + assert!( + matches!(tree.root().expr, NativeExpr::Window { .. }), + "the root stays the name-keeping window: {tree}" + ); +} + #[test] fn info_collects_the_side_that_aliases() { let out = grid(0, 1000, 2); @@ -780,7 +1494,7 @@ fn info_collects_the_side_that_aliases() { } #[test] -fn a_scalar_side_of_a_broadcast_is_not_collected() { +fn a_scalar_broadcast_collects_its_own_name_drop() { let out = grid(0, 1000, 2); let tree = passed( binary_over( @@ -790,7 +1504,23 @@ fn a_scalar_side_of_a_broadcast_is_not_collected() { ), &scan_tags(), ); - assert_eq!(0, merges(&tree), "a constant holds no label set"); + // `x + 2` drops the metadata labels, so what it hands out is collected - above the match, not + // beside it - and neither operand is collected on its own. + assert_eq!(1, merges(&tree), "{tree}"); + let NativeExpr::Merge { input } = &tree.root().expr else { + panic!("the root collects what the broadcast hands out: {tree}"); + }; + let NativeExpr::Binary { left, right, .. } = &input.expr else { + panic!("the root stays the broadcast"); + }; + assert!( + matches!(left.expr, NativeExpr::Scalar(_)), + "a constant holds no label set" + ); + assert!( + !matches!(right.expr, NativeExpr::Merge { .. }), + "the vector side hands its series over as it reads them" + ); } #[test] @@ -813,7 +1543,7 @@ fn aggregate_state_is_not_collected() { let tree = passed(state, &scan_tags()); assert_eq!(0, merges(&tree)); assert!(matches!(tree.root().output, NativeType::AggregateState(_))); - let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); assert_eq!(tree.to_string(), twice.to_string()); } @@ -837,7 +1567,8 @@ fn the_pass_is_idempotent() { ), &scan_tags(), ); - let twice = merge_label_rewrites(&once, &scan_tags()).expect("a collected tree is legal"); + let twice = + merge_label_rewrites(&once, &scan_tags(), &[], true).expect("a collected tree is legal"); assert_eq!(1, merges(&twice)); assert_eq!(once.to_string(), twice.to_string()); } diff --git a/src/promql/src/native/execute.rs b/src/promql/src/native/execute.rs index 55c81d5c664..8421e6f4052 100644 --- a/src/promql/src/native/execute.rs +++ b/src/promql/src/native/execute.rs @@ -18,8 +18,7 @@ //! tree builds independent executions. A caller registers the tree's per-node metric handles into //! its own metrics set once, before the execution starts. -use std::collections::hash_map::Entry; -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{HashSet, VecDeque}; use std::sync::Arc; use common_query::promql_annotations::PromqlAnnotationCollector; @@ -38,8 +37,8 @@ use datatypes::series::{SeriesLabels, SeriesOrdering}; use crate::aggregate; use crate::grid_series::{ - GridSample, GridSeriesBuilder, LabelSet, fields_bytes, label_set, merge_same_labelset, owned, - series_bytes, table_bytes, + GridSample, GridSeriesBuilder, fields_bytes, merge_same_labelset, owned, series_bytes, + table_bytes, }; use crate::histogram_eval::float_histogram::FloatHistogram; use crate::histogram_eval::{ @@ -64,9 +63,6 @@ use crate::native::plan::{ BoundInstant, BoundScalar, BoundSetOp, BoundVectorOp, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, ParameterizedOp, unifies_its_own_contributions, }; -#[cfg(test)] -pub(crate) use crate::native::runs::absorb_rebuilds; -use crate::native::runs::{Run, absorb}; use crate::native::state::{StateChunk, state_bytes}; use crate::range_eval::RangeEvaluator; use crate::select::{self, SortOrder}; @@ -540,6 +536,15 @@ impl MergeInput { pub(super) struct NodeBuffer { metrics: Arc, reservation: MemoryReservation, + /// What this node keeps for the grid timestamps it materialized, folded into every + /// re-estimate of its own figure; see [`NodeBuffer::reserve_grid`]. + /// + /// `None` means this node has not reserved a grid yet and `Some(bytes)` that it has: what the + /// record says is that the reservation was made, which is not the same statement as the figure + /// it ended at. A grid always has at least one point today -- `GridSpec` refuses a length of + /// zero -- so the two happen to coincide, but a reservation of no bytes would still be a + /// reservation made, and reading the figure would make it one made per call. + grid: Option, } impl NodeBuffer { @@ -556,9 +561,46 @@ impl NodeBuffer { Self { metrics, reservation: consumer.register(pool), + grid: None, } } + /// Reserves the timestamps of `grid` before this node reads them, once for the execution. + /// + /// An [`EvalGrid`] allocates its timestamps on first access and keeps them for as long as the + /// plan holds the grid, which outlives the chunk a node read them for. The node that triggers + /// that allocation therefore reserves the array and keeps the reservation: giving it back with + /// the workspace would leave the execution holding bytes no reservation covers, which is what a + /// grid of a million deferred points -- the reason the array is deferred at all -- would go + /// unnoticed through. Only the node that materializes it reserves it: a grid that is already + /// materialized, whether the plan built it eagerly or a node below or beside this one needed it + /// first, was charged to whoever materialized it, so this node grows by nothing. + /// + /// The array is reserved once per node, not once per call: a node is handed its input one chunk + /// at a time and reaches the grid on every chunk it is handed, and a collecting window reaches + /// it again when it flushes what it collected -- which is the call that reads the timestamps, + /// because until then nothing on this node looks at them. Reserving again on each of those + /// calls charges the pool one array per chunk for the one array the execution holds, so a series + /// arriving in `n` pieces would need `n + 1` arrays' worth of budget and exhaust a pool sized + /// for the query over a chunk count that says nothing about what the node keeps. What short + /// circuits the later calls is therefore the reservation this node made (`grid`), not the array + /// having been materialized: it cannot stand in for it, being false exactly while a collecting + /// node is charging for it. + pub(super) fn reserve_grid(&mut self, grid: &EvalGrid) -> Result<()> { + if self.grid.is_some() || grid.is_materialized() { + return Ok(()); + } + let len = usize::try_from(grid.spec().len).map_err(|_| { + DataFusionError::Execution("grid length does not fit memory size".to_string()) + })?; + let bytes = len.checked_mul(std::mem::size_of::()).ok_or_else(|| { + DataFusionError::Execution("grid timestamp size overflow".to_string()) + })?; + self.grow(bytes)?; + self.grid = Some(bytes); + Ok(()) + } + /// Reserves `bytes` beside what this node already holds. pub(super) fn grow(&mut self, bytes: usize) -> Result<()> { self.reservation.try_grow(bytes)?; @@ -567,9 +609,12 @@ impl NodeBuffer { } /// Reserves exactly `bytes`, which is what this node holds from now on: a node that consumes its - /// input chunk by chunk is set from the state that is left, not grown per chunk. + /// input chunk by chunk is set from the state that is left, not grown per chunk. What a grid's + /// array takes is not part of that figure: the node keeps it for as long as it lives, so it + /// stays reserved on top of what the caller measured. pub(super) fn resize(&mut self, bytes: usize) -> Result<()> { - self.reservation.try_resize(bytes)?; + self.reservation + .try_resize(bytes.saturating_add(self.grid.unwrap_or(0)))?; self.record(); Ok(()) } @@ -612,51 +657,7 @@ impl Drop for NodeBuffer { } } -/// The positions one node has handed out, per output label set and field. -/// -/// A node that rewrites labels -- a `Map` dropping the metric name, a `Window` dropping the name and the -/// empty-valued labels -- can turn two input series into one label set, and the node above would read the -/// two chunks as one series. Recording which positions of each field were already handed out lets a second -/// series onto a set be refused before it is evaluated. -/// -/// Occupancy is per field and per position, and a chunk holding no value occupies nothing. A set is never -/// retired: `is_last` closes a source series, not a label set. The table is one node's own. -#[derive(Default)] -struct Emitted { - sets: HashMap, - /// How many run vectors, runs and label payload bytes every entry holds, folded in as the entries - /// change rather than by walking them, so [`Emitted::bytes`] is read in constant time. - counts: Counts, -} - -/// What the entries of an [`Emitted`] table hold, as running totals. -#[derive(Default)] -struct Counts { - /// The run vectors, one per field of one entry. - vectors: usize, - /// The occupied runs in every vector. - runs: usize, - /// The payload of every label set: the bytes of its names and values with one row per label. - payload: usize, -} - -/// What one node has handed out under one output label set. -#[derive(Default)] -struct History { - /// The occupied runs of each field, in position order and disjoint. - fields: Vec>, -} - -/// The bytes a label set holds: each label's name and value with the pair that refers to them. -fn label_bytes(labels: &LabelSet) -> usize { - labels - .iter() - .map(|(name, value)| std::mem::size_of::<(Arc, Arc)>() + name.len() + value.len()) - .sum() -} - -/// The bytes a metric name holds: its name and each tag's name and value, one row per label, which is -/// the convention [`label_bytes`] keeps. +/// The bytes a metric name holds: its name and each tag's name and value, one row per label. fn metric_bytes(metric: &SeriesLabels) -> usize { metric.name().map_or(0, |name| name.len()) + std::mem::size_of_val(metric.tags()) @@ -667,137 +668,6 @@ fn metric_bytes(metric: &SeriesLabels) -> usize { .sum::() } -impl Emitted { - /// Records what `chunk` occupies, rejecting it against what this node already handed out. - /// - /// Two values of one field at one position is the only thing refused: a second series that fills - /// only the positions the first one left empty merges into it. - fn occupy(&mut self, chunk: &EvaluatedChunk) -> Result<()> { - let labels = label_set(&chunk.series.metric); - let occupied: Vec> = chunk - .series - .fields - .iter() - .enumerate() - .map(|(index, field)| occupied_runs(index, field, &chunk.series.time)) - .collect::>()?; - let empty = occupied.iter().all(Vec::is_empty); - if let Some(history) = self.sets.get(&labels) { - for (index, runs) in occupied.iter().enumerate() { - let stored = history.fields.get(index).map_or(&[][..], Vec::as_slice); - for run in runs { - // The stored runs are sorted and disjoint, so only the one holding `run.start` - // or the one just before it can share a position. - let at = stored.partition_point(|stored| stored.end <= run.start); - let previous = stored.get(at).filter(|stored| stored.overlaps(*run)); - if let Some(previous) = previous { - return Err(occupied_position(&labels, index, run, previous)); - } - } - } - } - if empty { - // A chunk holding no value records nothing and closes nothing: a series that answered - // empty leaves every earlier series on this label set exactly where it was. - return Ok(()); - } - let fields = chunk.series.fields.len(); - let history = match self.sets.entry(labels) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - self.counts.payload += label_bytes(entry.key()); - entry.insert(History::default()) - } - }; - if history.fields.len() < fields { - self.counts.vectors += fields - history.fields.len(); - history.fields.resize_with(fields, Vec::new); - } - for (index, runs) in occupied.into_iter().enumerate() { - let stored = &mut history.fields[index]; - let before = stored.len(); - absorb(stored, &runs); - self.counts.runs = self.counts.runs - before + stored.len(); - } - Ok(()) - } - - /// An estimate of the bytes this table holds, which the node reserves for. - /// - /// Read from running totals, so it is constant time; the map's buckets and a vector's slack are - /// left out. - fn bytes(&self) -> usize { - let slots = - self.sets.len() * (std::mem::size_of::() + std::mem::size_of::()); - slots - + self.counts.vectors * std::mem::size_of::>() - + self.counts.runs * std::mem::size_of::() - + self.counts.payload - } -} - -/// The runs of positions holding a value in field `index`, as the axis `time` places its rows. -/// -/// A missing sample holds no value and a hole between rows is not a position, so a run ends where a hole -/// begins. Every field must hold one sample per row the axis places. -fn occupied_runs(index: usize, field: &Samples, time: &TimeAxis) -> Result> { - let mut runs: Vec = Vec::new(); - let mut open: Option = None; - let mut previous: Option = None; - let mut cursor = field.cursor(); - for (position, row) in time.positions() { - // The axis names the absolute position of every row, so the hole between two rows is - // visible as a jump in `position` while no row of this chunk sits in it. - if let Some(previous) = previous - && position != previous + 1 - && let Some(start) = open.take() - { - runs.push(Run { - start, - end: previous + 1, - }); - } - let sample = cursor - .at(row) - .ok_or_else(|| misaligned_field(index, time, field))?; - let present = !matches!(sample, SampleRef::Missing); - match (present, open) { - (true, None) => open = Some(position), - (false, Some(start)) => { - runs.push(Run { - start, - end: position, - }); - open = None; - } - _ => {} - } - previous = Some(position); - } - if let Some(start) = open { - // A run still open at the last row ends just past it, not where the field's row count says: - // the rows need not be adjacent to the positions above them. - runs.push(Run { - start, - end: previous.map_or(start, |last| last + 1), - }); - } - Ok(runs) -} - -/// The rejection of a series whose axis and fields disagree on how many rows there are. -/// -/// [`Series`] keeps its fields public, so a struct-literal chunk can carry fewer samples than the -/// axis places rows. -fn misaligned_field(index: usize, time: &TimeAxis, field: &Samples) -> DataFusionError { - DataFusionError::Execution(format!( - "a series was handed to the node whose axis and fields are misaligned: the axis places {} \ - rows, but field {index} holds {} samples, so the axis names a row the field does not have", - time.len(), - field.len(), - )) -} - /// What a map node applies. struct MapRuntime { ops: Arc<[MapOp]>, @@ -2194,9 +2064,11 @@ impl BroadcastRuntime { /// Whether the operation rewrites the label set of what it hands out. /// - /// Arithmetic and a `bool` comparison drop the metric name, so two input series can become one label - /// set under this node; a filtering comparison keeps the label set it read. The answer is read off - /// [`Self::op`], so an operation and its label policy cannot disagree. + /// Arithmetic and a `bool` comparison drop the metric name, so two input series can become one + /// label set under this node; a filtering comparison keeps the label set it read. The answer is + /// read off [`Self::op`], so an operation and its label policy cannot disagree. Collection is not + /// this node's work: a collision the rewritten set creates is what a canonical + /// [`NativeExpr::Merge`] above it refuses. fn drops_labels(&self) -> bool { match self.op { BroadcastOp::Arithmetic(_) | BroadcastOp::Compare(Comparison::Bool(_)) => true, @@ -2794,11 +2666,10 @@ enum RuntimeOp { /// State shared by the stateful operators. /// -/// The queue and emitted history must precede the buffer so they drop before its reservation. +/// The queue must precede the buffer so it drops before its reservation. struct NodeState { queued: VecDeque<(NativeChunk, usize)>, queued_tables: usize, - emitted: Option, buffer: NodeBuffer, } @@ -2892,7 +2763,6 @@ impl NodeState { Self { queued: VecDeque::new(), queued_tables: 0, - emitted: None, buffer: NodeBuffer::new(name, metrics, pool), } } @@ -2919,6 +2789,7 @@ impl RuntimeNode { binary_before: &mut usize, location: String, duplicate_evidence: Option>, + sources: &SourceFacts, ) -> Result { stacker::maybe_grow(NATIVE_STACK_RED_ZONE, NATIVE_STACK_GROWTH, || { Self::build_within_stack( @@ -2928,6 +2799,7 @@ impl RuntimeNode { binary_before, location, duplicate_evidence, + sources, ) }) } @@ -2939,6 +2811,7 @@ impl RuntimeNode { binary_before: &mut usize, location: String, duplicate_evidence: Option>, + sources: &SourceFacts, ) -> Result { let metrics = Arc::new(NodeMetrics::default()); let op = match &node.expr { @@ -2985,6 +2858,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; RuntimeOp::ScalarOf { input: Box::new(child), @@ -3031,6 +2905,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; RuntimeOp::VectorOf { input: Box::new(child), @@ -3052,6 +2927,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let input_schema = input.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution( @@ -3062,15 +2938,11 @@ impl RuntimeNode { input: Box::new(child), instant: InstantRuntime::try_new(&input_schema, op)?, state: { - let mut state = NodeState::new( + NodeState::new( format!("native.instant at {location}"), Arc::clone(&metrics), pool, - ); - if op.value == InstantValue::Timestamp { - state.emitted = Some(Emitted::default()); - } - state + ) }, } } @@ -3082,6 +2954,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let schema = node.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("merge does not produce samples".to_string()) @@ -3105,18 +2978,23 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let input_schema = input.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("window consumes non-sample input".to_string()) })?; - let merge = matches!(unifies_its_own_contributions(input), Some(false)) + // The child reports fragments only when it cannot prove the labels it drops tell its + // series apart, which is the same question the canonical pass asked before it + // decided this window needs no merge above it. + let unification = + unifies_its_own_contributions(input, &sources.tags, &sources.metrics); + let merge = matches!(unification, Some(false)) .then(|| MergeInput::new(Arc::clone(&input_schema))); - let mut state = NodeState::new( + let state = NodeState::new( format!("native.window at {location}"), Arc::clone(&metrics), pool, ); - state.emitted = Some(Emitted::default()); RuntimeOp::Window { input: Box::new(child), // The collector arrives through `set_annotations` before the first pull; @@ -3139,16 +3017,16 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let schema = node.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("map does not produce samples".to_string()) })?; - let mut state = NodeState::new( + let state = NodeState::new( format!("native.map at {location}"), Arc::clone(&metrics), pool, ); - state.emitted = Some(Emitted::default()); RuntimeOp::Map { input: Box::new(child), map: MapRuntime { @@ -3170,6 +3048,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; // A function of two scalars answers a scalar, which has one unlabelled field on // the grid it is evaluated on; every other function here answers its input's own @@ -3208,15 +3087,15 @@ impl RuntimeNode { binary_before, format!("{location}.parameter{index}"), duplicate_evidence.clone(), + sources, ) }) .collect::>>()?; - let mut state = NodeState::new( + let state = NodeState::new( format!("native.map at {location}"), Arc::clone(&metrics), pool, ); - state.emitted = Some(Emitted::default()); RuntimeOp::ParameterizedMap { input: Box::new(child), parameters: nodes, @@ -3240,16 +3119,16 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let schema = node.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("timestamp does not produce samples".to_string()) })?; - let mut state = NodeState::new( + let state = NodeState::new( format!("native.timestamp at {location}"), Arc::clone(&metrics), pool, ); - state.emitted = Some(Emitted::default()); RuntimeOp::Timestamp { input: Box::new(child), schema, @@ -3288,6 +3167,7 @@ impl RuntimeNode { binary_before, format!("{location}.left"), duplicate_evidence.clone(), + sources, )?; let right_node = Self::build( right, @@ -3296,6 +3176,7 @@ impl RuntimeNode { binary_before, format!("{location}.right"), duplicate_evidence.clone(), + sources, )?; match (binary, broadcast) { (Some(binary), None) => RuntimeOp::Binary { @@ -3309,18 +3190,11 @@ impl RuntimeNode { ), }, (None, Some(broadcast)) => { - let vector = if broadcast.vector_left { left } else { right }; - let mut state = NodeState::new( + let state = NodeState::new( format!("native.binary at {location}"), Arc::clone(&metrics), pool, ); - // Only an operation that rewrites the label set is where two input series - // can become one, and only one that makes an identity its vector side has - // not already checked needs a table of its own. - if broadcast.drops_labels() && !elides_emitted_table(&broadcast, vector) { - state.emitted = Some(Emitted::default()); - } RuntimeOp::Broadcast { left: Box::new(left_node), right: Box::new(right_node), @@ -3358,6 +3232,7 @@ impl RuntimeNode { binary_before, format!("{location}.left"), duplicate_evidence.clone(), + sources, )?; let right_node = Self::build( right, @@ -3366,6 +3241,7 @@ impl RuntimeNode { binary_before, format!("{location}.right"), duplicate_evidence.clone(), + sources, )?; RuntimeOp::Binary { left: Box::new(left_node), @@ -3405,6 +3281,7 @@ impl RuntimeNode { binary_before, format!("{location}.base"), duplicate_evidence.clone(), + sources, )?; let info_node = Self::build( info, @@ -3413,6 +3290,7 @@ impl RuntimeNode { binary_before, format!("{location}.info"), duplicate_evidence.clone(), + sources, )?; RuntimeOp::Binary { left: Box::new(base_node), @@ -3447,6 +3325,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; RuntimeOp::Aggregate { input: Box::new(child), @@ -3476,6 +3355,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let parameter = Self::build( parameter, @@ -3484,6 +3364,7 @@ impl RuntimeNode { binary_before, format!("{location}.parameter"), duplicate_evidence.clone(), + sources, )?; let state = NodeState::new( format!("native.vector at {location}"), @@ -3538,6 +3419,7 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let mut built = Vec::with_capacity(parameters.len()); for (index, parameter) in parameters.iter().enumerate() { @@ -3548,6 +3430,7 @@ impl RuntimeNode { binary_before, format!("{location}.parameter{index}"), duplicate_evidence.clone(), + sources, )?; built.push(Some(runtime)); } @@ -3591,9 +3474,15 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), + sources, )?; let limitk = matches!(op, BoundVectorOp::LimitK { .. }); - if limitk && matches!(unifies_its_own_contributions(input), Some(false)) { + if limitk + && matches!( + unifies_its_own_contributions(input, &sources.tags, &sources.metrics), + Some(false) + ) + { vector.merge_input = true; } RuntimeOp::Vector { @@ -3950,6 +3839,12 @@ impl RuntimeNode { state, } => { if !std::mem::replace(produced, true) { + // `time()` and the date functions read every point of the grid, so this node is + // what materializes a deferred grid's timestamps. + state + .buffer + .reserve_grid(grid) + .map_err(|error| state.context(error))?; let schema = scalar_schema(Arc::clone(grid)); state .produce(grid, || match date { @@ -4373,7 +4268,7 @@ impl RuntimeNode { .settle_vector(operation) .map_err(|error| state.context(error))?; // Finish windows before dropping the consumed subtree; its children may still - // hold charged emitted history or scalar series even after the scalar was read. + // hold charged buffers or scalar series even after the scalar was read. parameter_node.finish_windows()?; *parameter = None; } @@ -4906,9 +4801,13 @@ impl RuntimeNode { .. } => { let child = input.finish_windows(); + // Finishing evaluates the points still owed, which reads the grid's timestamps. let own = window .take() - .map_or(Ok(()), RangeEvaluator::finish) + .map_or(Ok(()), |evaluator| { + state.buffer.reserve_grid(evaluator.grid())?; + RangeEvaluator::finish(evaluator) + }) .map_err(|error| state.context(error)); child.and(own) } @@ -4916,11 +4815,59 @@ impl RuntimeNode { } } +/// What a fused execution knows about the labels its inputs hand over. +/// +/// The planner binds the tree's inputs and knows two things about each: the complete label set it +/// reports, and - when the statement reads one table - the one metric every series it hands over is +/// named. [`unifies_its_own_contributions`] reads exactly those facts, so the collection a consumer +/// performs at run time is decided by the same proof the canonical pass placed its merges with. A +/// caller that knows nothing sets no facts, and every consumer then collects conservatively. +#[derive(Clone, Debug, Default)] +pub struct SourceFacts { + /// The complete runtime label set of each input, indexed by the `source_index` of its leaf. + tags: Vec>, + /// The one metric each input is bound to, indexed the same way; `None` when the input may hand + /// over several, which is what a statement reading several tables binds. + metrics: Vec>>, +} + +impl SourceFacts { + /// Says nothing about any input, so every decision stays conservative. + pub fn none() -> Self { + Self::default() + } + + /// One statement-level fact, repeated for each of `sources` inputs. + /// + /// A binding names one metric and one tag set for the node as a whole, and every scan below it + /// reads that one table, so the fact holds for each of them. A binding that names no metric - + /// the node reads one raw source per input - proves nothing, and the proof fails with it. + pub fn uniform(tags: Vec, metric: Option>, sources: usize) -> Self { + Self { + tags: vec![tags; sources], + metrics: vec![metric; sources], + } + } +} + impl NativeExecutionState { pub fn try_new( tree: &NativeTree, pool: Arc, duplicate_evidence: Option>, + ) -> Result { + Self::try_new_with_sources(tree, pool, duplicate_evidence, &SourceFacts::none()) + } + + /// Builds the execution over `tree`, judging a dropped label set by `sources`. + /// + /// The canonical pass placed its merges with the same facts, so a consumer that collects here + /// collects exactly where the pass decided a consumer has to - and nowhere else. + pub fn try_new_with_sources( + tree: &NativeTree, + pool: Arc, + duplicate_evidence: Option>, + sources: &SourceFacts, ) -> Result { let mut inputs = Vec::new(); let mut binary_before = 0usize; @@ -4931,6 +4878,7 @@ impl NativeExecutionState { &mut binary_before, "root".to_string(), duplicate_evidence.clone(), + sources, )?; // A constant root has no node that produces its value, so the state produces it. let scalar_root = match (&tree.root().expr, &tree.root().output) { @@ -5144,21 +5092,18 @@ impl NodeState { let output = match outcome { Ok(output) => output, Err(error) => { - self.buffer.resize(self.emitted_bytes())?; + self.buffer.resize(0)?; return Err(error); } }; if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - // What it handed out is what the label sets of this node now hold: `map_chunk` has - // already rewritten the output's label set, so the label set of the output is the one - // two input series can share. - self.track(output)?; } - // The chunk is gone and the label sets this node has handed out are what is left. - let emitted = self.emitted_bytes(); - self.buffer.resize(emitted)?; + // The chunk is gone and this node holds none of it: a rewritten label set is not this + // node's to remember, and what two series sharing one would mean is a canonical merge's + // question. + self.buffer.resize(0)?; Ok(output) } @@ -5188,8 +5133,7 @@ impl NodeState { let outcome = match outcome { Ok(outcome) => outcome, Err(error) => { - self.buffer - .resize(self.emitted_bytes().saturating_add(map.parameter_bytes))?; + self.buffer.resize(map.parameter_bytes)?; return Err(error); } }; @@ -5207,11 +5151,9 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - self.track(output)?; } - let emitted = self.emitted_bytes(); - self.buffer - .resize(emitted.saturating_add(map.parameter_bytes))?; + // What this node still holds is what it read for its parameters and nothing else. + self.buffer.resize(map.parameter_bytes)?; Ok(output) } @@ -5223,21 +5165,16 @@ impl NodeState { ) -> Result> { let positions = chunk.series.time.len(); let closing = chunk.is_last; - // The kernel reads the grid's timestamps, which it materializes when they are deferred. - let grid_points = schema.eval_grid.as_ref().map_or(0, |grid| { - usize::try_from(grid.spec().len).unwrap_or(usize::MAX) - }); + // The kernel reads the grid's timestamps, which materializes them when the grid is + // deferred. They stay materialized for the rest of the execution, so this node reserves + // them beside what it holds and keeps them there rather than treating them as a workspace + // it gives back. + if let Some(grid) = schema.eval_grid.as_deref() { + self.buffer.reserve_grid(grid)?; + } let workspace = built_series_workspace_bytes(&chunk.series)?; - let scratch = grid_points - .checked_mul(std::mem::size_of::()) - .and_then(|timestamps| timestamps.checked_add(workspace)) - .ok_or_else(|| { - DataFusionError::Execution( - "the workspace of timestamp does not fit one address space".to_string(), - ) - })?; self.buffer.grow(evaluated_bytes(&chunk))?; - self.buffer.grow(scratch)?; + self.buffer.grow(workspace)?; let labels = drop_metric_name(&chunk.series.metric)?; let metrics = Arc::clone(&self.buffer.metrics); let result = { @@ -5252,7 +5189,7 @@ impl NodeState { let result = match result { Ok(result) => result, Err(error) => { - self.buffer.resize(self.emitted_bytes())?; + self.buffer.resize(0)?; return Err(error); } }; @@ -5264,9 +5201,8 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - self.track(output)?; } - self.buffer.resize(self.emitted_bytes())?; + self.buffer.resize(0)?; Ok(output) } @@ -5313,8 +5249,7 @@ impl NodeState { let output = match outcome { Ok(output) => output, Err(error) => { - self.buffer - .resize(self.emitted_bytes().saturating_add(broadcast.scalar_bytes))?; + self.buffer.resize(broadcast.scalar_bytes)?; return Err(error); } }; @@ -5325,17 +5260,9 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - // Only an operation that rewrites the label set is where two input series can become one, and - // only that one keeps the table `track` records into. - if broadcast.drops_labels() { - self.track(output)?; - } } - // The chunk is gone: the label sets this node has handed out and the scalar's value are - // what is left. - let emitted = self.emitted_bytes(); - self.buffer - .resize(emitted.saturating_add(broadcast.scalar_bytes))?; + // The chunk is gone and the scalar's value is what is left. + self.buffer.resize(broadcast.scalar_bytes)?; Ok(output) } @@ -5401,6 +5328,9 @@ impl NodeState { merge: &mut Option, chunk: NativeChunk, ) -> Result> { + // A window reads the grid's timestamps to place every point it decides, which materializes + // a deferred grid; the array stays materialized, so the reservation for it is kept. + self.buffer.reserve_grid(window.grid())?; if let Some(merge) = merge.as_mut() { let NativeChunk::Evaluated(chunk) = chunk else { return Err(DataFusionError::Execution( @@ -5461,22 +5391,19 @@ impl NodeState { let output = match outcome { Ok(output) => output, Err(error) => { - self.buffer - .resize(window.retained_bytes().saturating_add(self.emitted_bytes()))?; + self.buffer.resize(window.retained_bytes())?; return Err(error); } }; if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - self.track(output)?; } - // The chunk has been consumed: what this node still holds is what the evaluator kept of it and the - // label sets it has already handed out. Neither is charged per chunk consumed, which would make a - // fixed window over a long series read as an allocation failure. - let retained = window.retained_bytes(); - let emitted = self.emitted_bytes(); - self.buffer.resize(retained.saturating_add(emitted))?; + // The chunk has been consumed: what this node still holds is what the evaluator kept of it, + // charged whether or not this chunk produced an answer: a chunk that yields nothing leaves + // the evaluator's samples where they were, and the figure is not charged per chunk consumed, + // which would make a fixed window over a long series read as an allocation failure. + self.buffer.resize(window.retained_bytes())?; Ok(output) } @@ -5766,6 +5693,11 @@ impl NodeState { )); }; let (positions, closing) = (chunk.time.len(), chunk.is_last); + // A selection that answers with timestamps reads every point of the grid, which materializes + // a deferred grid's array; the reservation is kept because the array is. + if instant.selection.value != InstantValue::Sample { + self.buffer.reserve_grid(&instant.selection.output_grid)?; + } // The chunk is held from here on: a selection keeps every chunk of a series until the one // that closes it, so what this node reserves for is the series it has buffered. self.buffer.grow(raw_bytes(&chunk))?; @@ -5804,9 +5736,6 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - if instant.selection.value == InstantValue::Timestamp { - self.track(output)?; - } } // What this node still holds is the chunks of the series it is selecting: a whole series is held // until it ends. The chunk just consumed and the gather's workspace are both gone, so this gives @@ -5817,11 +5746,7 @@ impl NodeState { /// Sets this node's reservation to the chunks of the series it is still gathering. fn settle_buffered(&mut self, instant: &InstantRuntime) -> Result<()> { - self.buffer.resize( - instant - .buffered_bytes() - .saturating_add(self.emitted_bytes()), - ) + self.buffer.resize(instant.buffered_bytes()) } /// Retains and validates a fragment for canonicalization. @@ -5878,12 +5803,9 @@ impl NodeState { chunk: EvaluatedChunk, ) -> Result> { self.collect_merge(merge, chunk)?; - // What this node holds is the fragments and the label sets it has already handed out; its - // evaluator has read none of them yet. + // What this node holds is the fragments; its evaluator has read none of them yet. let retained = window.retained_bytes(); - let emitted = self.emitted_bytes(); - self.buffer - .resize(retained.saturating_add(emitted).saturating_add(merge.bytes))?; + self.buffer.resize(retained.saturating_add(merge.bytes))?; Ok(None) } @@ -5905,6 +5827,8 @@ impl NodeState { // evaluator never read a series and the charge goes back. return self.buffer.shrink(merge.bytes); } + // The merged series is evaluated by the window, which reads the grid's timestamps. + self.buffer.reserve_grid(window.grid())?; // The workspace the merge allocates is charged before it runs, from the rows, segments and // fields of the fragments as they were collected. let workspace = merge.workspace_bytes(); @@ -5945,15 +5869,9 @@ impl NodeState { .metrics .record_grid_points(output.series.time.len()); self.buffer.metrics.record_output(1, valid_values(&output)); - self.track(&output)?; let retained = window.retained_bytes(); - let emitted = self.emitted_bytes(); - self.buffer.resize( - retained - .saturating_add(emitted) - .saturating_add(held) - .saturating_add(queued_bytes), - )?; + self.buffer + .resize(retained.saturating_add(held).saturating_add(queued_bytes))?; let bytes = evaluated_bytes(&output); self.enqueue_output(output)?; queued_bytes = queued_bytes.saturating_add(bytes); @@ -5991,26 +5909,6 @@ impl NodeState { } Ok(()) } - - /// Records what this node hands out under its output label set. - /// - /// A node that rewrites labels is where two input series become indistinguishable, so this is where a - /// second one is refused: checking at the root would be too late for a window above, which would - /// already have evaluated the two as two series. - fn track(&mut self, output: &EvaluatedChunk) -> Result<()> { - match self.emitted.as_mut() { - Some(emitted) => emitted.occupy(output), - // A node that keeps its input's labels cannot make two series one. - None => Ok(()), - } - } - - /// The bytes the label sets this node has handed out hold. - /// - /// Zero for a node that keeps its input's labels, which keeps no such table. - fn emitted_bytes(&self) -> usize { - self.emitted.as_ref().map_or(0, Emitted::bytes) - } } impl InputSlot { @@ -6468,24 +6366,6 @@ fn missing_row(index: usize) -> DataFusionError { )) } -/// The rejection of a chunk holding a value where the same field already holds one. -/// -/// No merge resolves this one: at `position`, field `field` of the label set `labels` would carry two -/// values, which is what Prometheus rejects for two series whose label sets became equal. -fn occupied_position( - labels: &LabelSet, - field: usize, - run: &Run, - previous: &Run, -) -> DataFusionError { - DataFusionError::Execution(format!( - "vector cannot contain metrics with the same labelset {}: two series hold a value of field \ - {field} at position {position} ({previous:?} and {run:?})", - crate::grid_series::render_label_set(labels), - position = run.start.max(previous.start), - )) -} - /// The rejection of an input that ended without the chunk closing its series. /// /// A tree with no window has no evaluator to hold the series, so the input node is the one node left that @@ -6619,20 +6499,6 @@ fn broadcast_runtime( }) } -/// Whether the [`Emitted`] table `broadcast` would keep only repeats the one its vector side keeps. -/// -/// Literal arithmetic leaves missing samples missing and floats at their positions, so its output occupancy -/// is a subset of the window's, and the second label drop removes only empty-valued labels that -/// [`label_set`] ignores. Arithmetic therefore cannot create a collision the window did not already refuse. -fn elides_emitted_table(broadcast: &BroadcastRuntime, vector: &NativeNode) -> bool { - matches!(broadcast.scalar, BroadcastScalar::Literal(_)) - && matches!(broadcast.op, BroadcastOp::Arithmetic(_)) - && matches!( - &vector.expr, - NativeExpr::Window { op, .. } if op.function.drops_metric_name() - ) -} - /// The operation a binary node's `matching` describes. /// /// A comparison is a filter unless it was bound as `bool`, and a filter keeps the label set it read while diff --git a/src/promql/src/native/execute/tests.rs b/src/promql/src/native/execute/tests.rs index d16241b3d41..d61e665e7b8 100644 --- a/src/promql/src/native/execute/tests.rs +++ b/src/promql/src/native/execute/tests.rs @@ -15,7 +15,7 @@ //! What the runtime has to get right: each layer's grid, per-node accounting, rejection and //! release. -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Weak}; use datafusion::error::DataFusionError; @@ -43,10 +43,10 @@ fn is_resources_exhausted(error: &DataFusionError) -> bool { } use super::{ - BinaryRuntime, Emitted, InputId, InputSlot, InstantRuntime, NativeChunk, NativeExecutionState, + BinaryRuntime, InputId, InputSlot, InstantRuntime, NativeChunk, NativeExecutionState, NativeExecutor, NodeState, PairKernel, Reader, RetainedInput, RuntimeNode, RuntimeOp, - absorb_rebuilds, buffered_fields, buffered_time, evaluated_bytes, grid_chunk_bytes, - grid_end_ms, grid_position, map_chunk, map_workspace_bytes, raw_bytes, vector_series_with_k, + SourceFacts, buffered_fields, buffered_time, evaluated_bytes, grid_chunk_bytes, grid_end_ms, + grid_position, map_chunk, map_workspace_bytes, raw_bytes, vector_series_with_k, vector_workspace_bytes, }; use crate::evaluated::EvaluatedChunk; @@ -58,6 +58,7 @@ use crate::instant_eval::{instant_candidate_span, select_instants}; use crate::map::MathOp; use crate::map_label::LabelRewrite; use crate::map_time::DateFunction; +use crate::native::canonicalize::merge_label_rewrites; use crate::native::metrics::{NodeMetric, NodeMetrics}; use crate::native::plan::{ AggregateKind, AggregateStage, AggregateStateSchema, BoundAggregate, BoundInput, BoundInstant, @@ -122,6 +123,93 @@ impl MemoryPool for DropOrderPool { } } +/// A pool that records every reservation it is asked to admit, so a test can count the calls that +/// made one rather than infer them from the figure a reservation ends at. +/// +/// The figure cannot tell two reservations of no bytes apart from one, and it cannot tell a +/// reservation made twice from one made once and then left alone at the same size; the calls can. +/// Each entry is the consumer's name and the bytes it asked for, in the order they were asked for, +/// and the largest total the pool was asked to hold at once is kept beside them: that figure is +/// what a limit has to admit for a run to fit, and it is a maximum inside a call, which the +/// reservation's size between calls does not show. +#[derive(Debug)] +struct GrowCallPool { + inner: GreedyMemoryPool, + calls: std::sync::Mutex>, + peak: AtomicUsize, +} + +impl std::fmt::Display for GrowCallPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "grow-call test pool") + } +} + +impl GrowCallPool { + fn new(bytes: usize) -> Self { + Self { + inner: GreedyMemoryPool::new(bytes), + calls: std::sync::Mutex::new(Vec::new()), + peak: AtomicUsize::new(0), + } + } + + /// The largest total this pool was asked to hold at one time. + fn peak(&self) -> usize { + self.peak.load(Ordering::SeqCst) + } + + /// The bytes the consumer called `name` asked this pool to admit, in order. + fn grows(&self, name: &str) -> Vec { + self.calls + .lock() + .unwrap() + .iter() + .filter(|(consumer, _)| consumer == name) + .map(|(_, bytes)| *bytes) + .collect() + } + + fn record(&self, reservation: &MemoryReservation, additional: usize) { + self.peak.fetch_max( + reservation.size().saturating_add(additional), + Ordering::SeqCst, + ); + self.calls + .lock() + .unwrap() + .push((reservation.consumer().name().to_string(), additional)); + } +} + +impl MemoryPool for GrowCallPool { + fn name(&self) -> &str { + "grow-call test pool" + } + + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.record(reservation, additional); + self.inner.grow(reservation, additional); + } + + fn try_grow( + &self, + reservation: &MemoryReservation, + additional: usize, + ) -> Result<(), DataFusionError> { + self.record(reservation, additional); + self.inner.try_grow(reservation, additional) + } + + fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { + self.inner.shrink(reservation, shrink); + } + + fn reserved(&self) -> usize { + self.inner.reserved() + } +} + fn grid(start: i64, step: i64, len: u64) -> Arc { Arc::new(EvalGrid::try_new(GridSpec::try_new(start, step, len).unwrap()).unwrap()) } @@ -335,7 +423,7 @@ fn instant( fn window_and_limitk_collect_only_unified_contributions() { let grid = grid(0, 1_000, 2); - let build = |node: NativeNode| { + let build_with = |node: NativeNode, sources: &SourceFacts| { let tree = NativeTree::try_new(node).unwrap(); let mut inputs = Vec::new(); let mut binary_before = 0; @@ -346,9 +434,11 @@ fn window_and_limitk_collect_only_unified_contributions() { &mut binary_before, "root".to_string(), None, + sources, ) .unwrap() }; + let build = |node: NativeNode| build_with(node, &SourceFacts::none()); let raw_window = build(window_with( evaluated_input(Arc::clone(&grid)), @@ -463,6 +553,54 @@ fn window_and_limitk_collect_only_unified_contributions() { !vector.merge_input, "limitk must not re-merge a window that already unifies its contributions" ); + + // A window that drops the metadata labels collects its child only when something it drops tells + // the child's series apart. The child here is `abs(x)`, which already dropped the name: a caller + // that cannot say which metric the scan reads collects, and one that says the scan reads a + // single metric whose tags carry neither `__type__` nor `__unit__` does not - the same proof the + // canonical pass used when it decided this window needs no merge above it. + let dropped_name_window = || { + window_with( + window_with( + map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]), + RangeFunction::Rate, + 300_000, + Some(Arc::clone(&grid)), + Arc::clone(&grid), + ), + RangeFunction::LastOverTime, + 300_000, + Some(Arc::clone(&grid)), + Arc::clone(&grid), + ) + }; + let RuntimeOp::Window { + merge: unknown_facts, + .. + } = build(dropped_name_window()).op + else { + panic!("expected window runtime node"); + }; + assert!( + unknown_facts.is_some(), + "a caller that knows nothing about the scan collects" + ); + let named = SourceFacts::uniform( + vec!["host".to_string(), "__name__".to_string()], + Some(Arc::from("some_metric")), + 1, + ); + let RuntimeOp::Window { + merge: proven_facts, + .. + } = build_with(dropped_name_window(), &named).op + else { + panic!("expected window runtime node"); + }; + assert!( + proven_facts.is_none(), + "a single metric's dropped name tells none of its series apart" + ); } #[test] @@ -1253,6 +1391,214 @@ fn eval_time_reports_seconds_and_dates_of_the_grid() { assert_eq!(pool.reserved(), 0); } +/// The bytes a grid's timestamps take, which is what an execution reserves for the array. +fn grid_array_bytes(grid: &EvalGrid) -> usize { + usize::try_from(grid.spec().len).unwrap() * std::mem::size_of::() +} + +/// A grid whose timestamps are still deferred, as a subquery's or an anchored step's is: the plan +/// is compiled without allocating them, and the execution that reads them allocates once. +fn deferred_grid(start: i64, step: i64, len: u64) -> Arc { + let grid = EvalGrid::try_deferred(GridSpec::try_new(start, step, len).unwrap()).unwrap(); + assert!(!grid.is_materialized()); + Arc::new(grid) +} + +/// The node that materializes a deferred grid reserves its array and keeps that reservation. +/// +/// A deferred grid is what keeps compilation proportional to the plan rather than to the timestamp +/// count, so the array it eventually needs is allocated at execution: the node that triggers the +/// allocation reserves it, and a pool that cannot hold it refuses the query instead of letting it +/// take the array unasked. What it reserved stays reserved for as long as the node lives, because +/// the array outlives the chunk the node read it for. +#[test] +fn a_node_that_materializes_a_deferred_grid_reserves_it_for_its_lifetime() { + let points = 1_000u64; + let grid = deferred_grid(0, 60_000, points); + let array = grid_array_bytes(&grid); + let tree = NativeTree::try_new(vector_of(eval_time(None, Arc::clone(&grid)))).unwrap(); + + // One byte short of the array: the node that would allocate it is refused instead. + let pool = bounded_pool(array - 1); + let mut execution = executor(&tree, Arc::clone(&pool)); + let error = execution.next_chunk().unwrap_err(); + assert!(is_resources_exhausted(&error), "{error}"); + assert!( + !grid.is_materialized(), + "a refused node materialized the grid anyway" + ); + assert_eq!(pool.reserved(), 0); + + // Room for the array and the chunk built from it: the query answers, and what the node keeps + // once its chunk has been handed on is the array alone. + let chunk = grid_chunk_bytes(&grid).unwrap(); + let pool = bounded_pool(array + chunk); + let mut execution = executor(&tree, Arc::clone(&pool)); + assert!(execution.next_chunk().unwrap().is_some()); + assert!( + grid.is_materialized(), + "the node did not materialize the grid" + ); + assert_eq!( + execution.state().reserved_bytes(), + array, + "the materialized grid was not kept reserved after its chunk was handed on" + ); + drop(execution); + assert_eq!( + pool.reserved(), + 0, + "the array's reservation outlived its node" + ); +} + +/// A grid that is already materialized is charged to nobody: the execution reads the array that is +/// there, and only the node that allocates it reserves it. +/// +/// This is also what keeps two nodes reading one grid from charging it twice, and what makes a grid +/// the plan built eagerly -- [`EvalGrid::try_new`] rather than [`EvalGrid::try_deferred`] -- cost +/// the execution nothing. +#[test] +fn a_materialized_grid_is_not_reserved_again() { + let points = 1_000u64; + let grid = grid(0, 60_000, points); + let array = grid_array_bytes(&grid); + assert!(grid.is_materialized()); + let chunk = grid_chunk_bytes(&grid).unwrap(); + let tree = NativeTree::try_new(vector_of(eval_time(None, Arc::clone(&grid)))).unwrap(); + // A pool one byte short of the array *and* the chunk: the deferred grid above needed both, and + // this one needs the chunk alone. + let pool = bounded_pool(array + chunk - 1); + let mut execution = executor(&tree, Arc::clone(&pool)); + assert!(execution.next_chunk().unwrap().is_some()); + assert_eq!(execution.state().reserved_bytes(), 0); + drop(execution); + assert_eq!(pool.reserved(), 0); +} + +/// A window reads the grid's timestamps to place every point it decides, so it materializes a +/// deferred output grid and keeps that reservation as well. +#[test] +fn a_window_reserves_the_grid_it_materializes() { + let points = 1_000u64; + let grid = deferred_grid(0, 60_000, points); + let array = grid_array_bytes(&grid); + let schema = raw_schema(); + let tree = NativeTree::try_new(window(input(), Arc::clone(&grid))).unwrap(); + + // A pool short of the array refuses the window instead of letting it take the array unasked. + let short = bounded_pool(array - 1); + let mut execution = executor(&tree, Arc::clone(&short)); + execution + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], true)) + .unwrap(); + let error = execution.next_chunk().unwrap_err(); + assert!(is_resources_exhausted(&error), "{error}"); + assert!( + !grid.is_materialized(), + "a refused window materialized the grid anyway" + ); + assert_eq!(short.reserved(), 0); + + // With room for the array the window keeps it while it holds the series it is evaluating. + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + execution + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) + .unwrap(); + drop(execution.next_chunk().unwrap()); + assert!( + grid.is_materialized(), + "the window did not materialize the grid" + ); + let held = execution.state().reserved_bytes(); + assert!( + held >= array, + "the window gave the grid's array back: {held} bytes held, {array} reserved for the grid" + ); + drop(execution); + assert_eq!(pool.reserved(), 0); +} + +/// A node that reads one grid reserves its timestamps once, however many chunks it is handed on +/// that grid. +/// +/// A collecting window reaches its output grid for every chunk it collects and again when it +/// flushes what it collected, and none of those calls finds the array materialized: the timestamps +/// are read by the flush, so what the calls before it reserve is an array that is still deferred. +/// What keeps the execution from charging one array per chunk is the node's record of the +/// reservation it made, which is what this pool counts, rather than the figure the reservation ends +/// at: a reservation the pool is asked for once is one call, whatever it is worth. +#[test] +fn reserve_grid_reserves_the_array_once_however_often_it_is_called() { + let grid = deferred_grid(0, 60_000, 1_000); + let array = grid_array_bytes(&grid); + let name = "native.window at root"; + let pool = Arc::new(GrowCallPool::new(1 << 20)); + let watched: Arc = Arc::clone(&pool) as Arc; + let mut state = NodeState::new(name, Arc::new(NodeMetrics::default()), &watched); + + // A collecting window's five chunks and the flush after them: six calls, one array. + for _ in 0..6 { + state.buffer.reserve_grid(&grid).unwrap(); + } + assert_eq!( + pool.grows(name), + vec![array], + "the node asked the pool to admit the array more than once" + ); + assert_eq!(state.buffer.reservation.size(), array); + assert!( + !grid.is_materialized(), + "reserving for the array read the timestamps the allocation is deferred for" + ); + + // A re-estimate carries the array on top of what it measured, and reaching the grid again is + // not a second reservation of it. + state.buffer.resize(128).unwrap(); + assert_eq!(state.buffer.reservation.size(), array + 128); + state.buffer.reserve_grid(&grid).unwrap(); + assert_eq!(pool.grows(name), vec![array, 128]); + + drop(state); + assert_eq!( + pool.reserved(), + 0, + "the array outlived the node it was reserved for" + ); +} + +/// A grid the plan already materialized is charged to nobody, however often a node reaches it. +/// +/// This is the other path a node takes through [`NodeBuffer::reserve_grid`](super::NodeBuffer): the +/// timestamps are there already, so the node that reads them allocates nothing and reserves +/// nothing, and what its own re-estimates add is its input alone. +#[test] +fn reserve_grid_charges_nothing_for_a_materialized_grid() { + let grid = grid(0, 60_000, 1_000); + assert!(grid.is_materialized()); + let name = "native.window at root"; + let pool = Arc::new(GrowCallPool::new(1 << 20)); + let watched: Arc = Arc::clone(&pool) as Arc; + let mut state = NodeState::new(name, Arc::new(NodeMetrics::default()), &watched); + + for _ in 0..4 { + state.buffer.reserve_grid(&grid).unwrap(); + } + assert!( + pool.grows(name).is_empty(), + "a node reserved for an array it did not allocate" + ); + assert_eq!(state.buffer.reservation.size(), 0); + + state.buffer.resize(64).unwrap(); + assert_eq!(pool.grows(name), vec![64]); + assert_eq!(state.buffer.reservation.size(), 64); + + drop(state); + assert_eq!(pool.reserved(), 0); +} + /// `vector(c)` hands the constant on as one unlabelled series over the whole grid. #[test] fn vector_of_a_constant_is_one_unlabelled_series() { @@ -3236,8 +3582,8 @@ fn two_series_onto_one_label_set_under_a_parameterized_topk_are_merged() { /// /// The pair below maps into one label set and both hold a value at position 0 of the field, which /// no merge resolves and Prometheus rejects for two series whose label sets became equal. The -/// refusal is the map's own, before the window above evaluates anything, and it names the position -/// rather than a closed label set: a closed source series is not a closed label set. +/// refusal is the merge's, which the window above runs on the input it collected before it evaluates +/// anything: nothing reaches the evaluator until the pieces of one label set are one series. #[test] fn two_series_onto_one_label_set_after_a_map_are_rejected() { let inner_grid = grid(0, 60_000, 2); @@ -3284,6 +3630,8 @@ fn two_series_onto_one_label_set_after_a_map_are_rejected() { ), ) .unwrap(); + // The window collects its input, so the two pieces are merged when the input ends. + execution.end_of_input().unwrap(); let error = match execution.next_chunk() { Ok(None) => panic!("the second series was accepted at a position the first one holds"), Ok(Some(chunk)) => panic!( @@ -3294,12 +3642,12 @@ fn two_series_onto_one_label_set_after_a_map_are_rejected() { }; let message = error.to_string(); assert!( - message.contains("job"), - "the rejection does not name the label set: {message}" + message.contains("vector cannot contain metrics with the same labelset"), + "the rejection does not name the defect: {message}" ); assert!( - message.contains("position 0"), - "the rejection does not name the shared position: {message}" + message.contains("job"), + "the rejection does not name the label set: {message}" ); } @@ -3508,10 +3856,9 @@ fn a_merge_flush_reserves_every_queued_output() { assert_eq!(first.series.metric.tags()[0].value.as_ref(), "a"); assert!(first.is_last, "a merged series is handed over closed"); - // What the root holds once one of the two answers left: the answer still queued, the label sets - // the node handed out under, and the evaluator that evaluated them. The borrow ends before the - // next pull. - let (queued, queued_bytes, retained, emitted, held) = { + // What the root holds once one of the two answers left: the answer still queued and the + // evaluator that evaluated it. The borrow ends before the next pull. + let (queued, queued_bytes, retained, held) = { let RuntimeOp::Window { window, state, .. } = &execution.state().live.as_ref().unwrap().root.op else { @@ -3522,7 +3869,6 @@ fn a_merge_flush_reserves_every_queued_output() { state.queued.len(), queued_bytes, window.as_ref().map_or(0, RangeEvaluator::retained_bytes), - state.emitted_bytes(), state.buffer.reservation.size(), ) }; @@ -3533,7 +3879,7 @@ fn a_merge_flush_reserves_every_queued_output() { ); assert_eq!( held, - queued_bytes + retained + emitted, + queued_bytes + retained, "the reservation after the first pull is not what the node still holds" ); assert_eq!(pool.reserved(), execution.state().reserved_bytes()); @@ -3669,6 +4015,173 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() { ); } +/// A `sum_over_time` window over a map over a source: the map can rewrite label sets, so the window +/// collects its whole input before it evaluates any of it. +/// +/// The range covers every point of the `points` the output grid has, so the series the window +/// eventually folds is the whole series it collected. +fn collecting_window_tree(grid: &Arc, points: u64) -> NativeTree { + let step = grid.spec().step_ms; + NativeTree::try_new(window_with( + map(evaluated_input(Arc::clone(grid)), vec![MathOp::Abs]), + RangeFunction::SumOverTime, + step.saturating_mul(points as i64), + Some(Arc::clone(grid)), + Arc::clone(grid), + )) + .unwrap() +} + +/// Delivers one series as `chunks` one-point chunks of `1.0` to a collecting window's source, +/// pulling after each of them. +/// +/// The chunks do not close the series except on the last one, so the window is still collecting +/// when the call returns and nothing has been answered: what changed is only what it keeps of the +/// chunks it was handed. +fn push_one_point_chunks(execution: &mut NativeExecutor, schema: &SeriesSchema, chunks: usize) { + for position in 0..chunks { + execution + .push_evaluated( + InputId::new(0), + evaluated_chunk(schema, position, vec![1.0], position + 1 == chunks), + ) + .unwrap(); + assert!( + execution.next_chunk().unwrap().is_none(), + "the collecting window answered at position {position} before its input ended" + ); + } +} + +/// A collecting window is handed its input one chunk at a time and reaches its output grid on every +/// one of them, so what it reserves for the timestamps must not grow with the chunk count. +/// +/// The window reads the grid's timestamps only when it flushes, so the array is still deferred when +/// the chunks arrive: a reservation made per call would charge the pool one array for every chunk +/// -- `chunks + 1` of them, the last one on the flush -- for the one array the execution holds. The +/// same chain over an eager grid charges nothing at all for the timestamps, because the plan +/// already allocated them, so what the two figures differ by is exactly what the deferred run +/// reserved: one array. +#[test] +fn a_collecting_window_reserves_its_grid_once_for_all_its_chunks() { + let points = 1_000u64; + let chunks = 64usize; + // What the chain holds once it has collected the series, for a grid whose array the plan + // built and for one whose array the flush has to build. + let held_after = |grid: Arc| { + let schema = evaluated_schema(Arc::clone(&grid)); + let tree = collecting_window_tree(&grid, points); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + push_one_point_chunks(&mut execution, &schema, chunks); + let held = execution.state().reserved_bytes(); + assert_eq!( + held, + pool.reserved(), + "the state and the pool disagree on what the collected input holds" + ); + let out = execution.finish().expect("the collected series evaluates"); + assert_eq!( + decided(&out) + .last() + .map(|(position, value)| (*position, *value)), + Some((points as usize - 1, Some(chunks as f64))), + "the window did not fold the whole collected series" + ); + assert_eq!(pool.reserved(), 0); + held + }; + + let deferred = deferred_grid(0, 60_000, points); + let array = grid_array_bytes(&deferred); + let eager_held = held_after(grid(0, 60_000, points)); + let deferred_held = held_after(Arc::clone(&deferred)); + assert_eq!( + deferred_held - eager_held, + array, + "the window reserved {} arrays for the one array it read: {deferred_held} bytes held over \ + a deferred grid against {eager_held} over an eager one, {array} per array", + (deferred_held - eager_held) / array + ); + assert!( + deferred.is_materialized(), + "the flush did not read the grid it answered on" + ); +} + +/// A pool sized for what a collecting window holds is not exhausted by the number of chunks it was +/// handed. +/// +/// The chain collects `chunks` one-point chunks of one series over a deferred grid, and the bytes it +/// needs are those fragments, the merge's workspace, the answer and one array of timestamps. The +/// same chain over an eager grid needs all of that and nothing for the timestamps, so its peak +/// plus a few arrays is a budget that holds the deferred run -- and a reservation made per call, +/// one array for each of `chunks + 1` calls, would run out of it after a handful of chunks. A +/// chunk count is then something the query carries rather than something it is sized by. +#[test] +fn many_chunks_do_not_exhaust_a_pool_that_holds_one_grid() { + let points = 1_000u64; + let chunks = 256usize; + let deferred = deferred_grid(0, 60_000, points); + let array = grid_array_bytes(&deferred); + let schema = evaluated_schema(Arc::clone(&deferred)); + + // What the same chain holds once it has collected the series, and the most its pool has to + // admit at one time, over an eager grid: the timestamps are the plan's, so neither figure has + // anything in it for them. + let eager = grid(0, 60_000, points); + let watched = Arc::new(GrowCallPool::new(1 << 20)); + let eager_pool: Arc = Arc::clone(&watched) as Arc; + let tree = collecting_window_tree(&eager, points); + let mut execution = executor(&tree, eager_pool); + push_one_point_chunks( + &mut execution, + &evaluated_schema(Arc::clone(&eager)), + chunks, + ); + let eager_held = execution.state().reserved_bytes(); + let out = execution + .finish() + .expect("the collected series evaluates over an eager grid"); + assert_eq!( + decided(&out) + .last() + .map(|(position, value)| (*position, *value)), + Some((points as usize - 1, Some(chunks as f64))), + "the window did not fold the whole collected series" + ); + let eager_peak = watched.peak(); + drop(execution); + assert_eq!(watched.reserved(), 0); + + // Room for what the eager chain needed and eight arrays: the one the deferred grid holds and + // slack for the chunk arriving. One array for every chunk does not fit in it. + let pool = bounded_pool(eager_peak + 8 * array); + let tree = collecting_window_tree(&deferred, points); + let mut execution = executor(&tree, Arc::clone(&pool)); + push_one_point_chunks(&mut execution, &schema, chunks); + assert_eq!( + execution.state().reserved_bytes(), + eager_held + array, + "the window reserved more than the one array it holds" + ); + assert_eq!(pool.reserved(), eager_held + array); + // The flush reads the timestamps, folds the collected series and hands the answer on, and a + // pool that holds the fragments, the workspace and the one array admits that too. + let out = execution.finish().expect("the collected series evaluates"); + assert_eq!( + decided(&out) + .last() + .map(|(position, value)| (*position, *value)), + Some((points as usize - 1, Some(chunks as f64))) + ); + assert!( + deferred.is_materialized(), + "the flush did not read the grid it answered on" + ); + assert_eq!(pool.reserved(), 0, "the release left bytes reserved"); +} + /// A pool that cannot admit what a collecting chain holds fails the execution and gives everything /// back. /// @@ -3819,93 +4332,99 @@ fn a_pool_that_cannot_admit_the_merge_workspace_releases_what_it_held() { ); } -/// A window that drops empty-valued labels cannot hand two series on under one label set either. +/// A window that drops empty-valued labels is collected when the scan's key columns can spell the +/// pair, and the merge above it rejects the two series as one label set. /// /// `{host="web-1", zone=""}` and `{host="web-1"}` are two series to the scan: an empty tag value is /// a present label, an absent one is not. `rate` drops the metric name and the empty-valued labels, -/// so both become `{host="web-1"}` and each covers the same grid; answering would hand the whole -/// grid out twice under the same labels and the same timestamps. No map is involved, so the -/// rejection has to be the node's own — a check at the root would be too late, because the window -/// above would already have evaluated the two as two series. +/// so both become `{host="web-1"}` and each covers the same grid. /// -/// The two cover the same positions of the same field, which is the collision no merge resolves: -/// the refusal names the position, not a closed label set, because closing a source series says -/// nothing about the label set it shared with another one. `rate` needs two samples, so neither -/// series decides position 0 -- it holds the single sample at its own timestamp -- and both decide -/// positions 1 to 4. The first position both occupy is therefore 1. +/// The one fact that tells the pass to collect the two is the scan's primary key - whether one of +/// its columns can hold `NULL` where another holds `''` - and this fixture states that it cannot +/// prove the pair apart, which is what a nullable tag column says. The window's output is therefore +/// collected, and the merge is where the second value of one label set at one position is refused: +/// Prometheus's answer, rather than two pieces of one series leaving unchecked. +/// +/// The merge is the canonical pass's decision, not this fixture's: the tree is built as the window +/// alone and handed to `merge_label_rewrites` with the facts a planner states - it reads one metric +/// and binds `host`, `zone` and `__name__`, and its key columns can hold that pair - so what is +/// exercised is the pass's answer. #[test] fn two_series_onto_one_label_set_after_a_window_are_rejected() { let out_grid = grid(0, 60_000, 5); - let tree = NativeTree::try_new(window_with( - input(), - RangeFunction::Rate, - 300_000, - None, - Arc::clone(&out_grid), - )) - .unwrap(); + // One tree per execution: a stopped execution keeps its state. + let tree = || { + merge_label_rewrites( + &NativeTree::try_new(window_with( + input(), + RangeFunction::Rate, + 300_000, + None, + Arc::clone(&out_grid), + )) + .unwrap(), + &[vec![ + "host".to_string(), + "zone".to_string(), + "__name__".to_string(), + ]], + &[Some(Arc::::from("requests"))], + // The scan keys its series by a column that can hold `NULL` in one series and `''` in + // another, so its labels alone cannot prove the drop. + false, + ) + .unwrap() + }; + let tree = tree(); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "the key columns cannot tell the pair apart, so the pass collects it: {tree}" + ); let schema = raw_schema(); let counter = [0, 60_000, 120_000, 180_000, 240_000]; - let mut execution = executor(&tree, pool()); - execution - .push_raw( - InputId::new(0), - raw_chunk_of( - &schema, - b"series-a", - host_metric(Some("")), - &counter, - &[1.0, 2.0, 3.0, 4.0, 5.0], - true, - ), - ) - .unwrap(); - let first = execution - .next_chunk() - .unwrap() - .expect("the first series settles the grid"); - // The decided positions of the first series, which are the ones the second one has to share. - assert_eq!( - first.series.time.runs().count(), - 1, - "the first series decided more than one run" - ); - assert_eq!( - first.series.time.first_position(), - Some(1), - "`rate` decided a position the single sample at 0 cannot settle" - ); - assert!(first.is_last); - // The second series is a different one to the scan and the same one to `rate`. - execution - .push_raw( - InputId::new(0), - raw_chunk_of( - &schema, - b"series-b", - host_metric(None), - &counter, - &[6.0, 7.0, 8.0, 9.0, 10.0], - true, - ), - ) - .unwrap(); - let error = match execution.next_chunk() { - Ok(None) => panic!("the second series settled nothing at all"), - Ok(Some(chunk)) => panic!( - "a second series left the window under the first one's label set: {:?}", - decided(&[chunk]) + let mut execution = executor(&tree, pool()); + for (key, metric, values) in [ + ( + b"series-a".as_slice(), + host_metric(Some("")), + [1.0, 2.0, 3.0, 4.0, 5.0], ), - Err(error) => error, - }; + ( + b"series-b".as_slice(), + host_metric(None), + [6.0, 7.0, 8.0, 9.0, 10.0], + ), + ] { + execution + .push_raw( + InputId::new(0), + raw_chunk_of(&schema, key, metric, &counter, &values, true), + ) + .unwrap(); + } + execution.end_of_input().unwrap(); + let mut rejected = None; + loop { + match execution.next_chunk() { + Ok(None) => break, + // The first piece may settle before the second collides with it; what must not happen is + // both leaving, and neither is the answer the merge owes this pair. + Ok(Some(_)) => {} + Err(error) => { + rejected = Some(error); + break; + } + } + } + let error = rejected.unwrap_or_else(|| { + panic!("the merge handed two pieces of one label set over instead of refusing the pair") + }); let message = error.to_string(); - assert!(message.contains("host"), "{message}"); - assert!(message.contains("web-1"), "{message}"); assert!( - message.contains("position 1"), - "the rejection does not name the first position both series hold: {message}" + message.contains("vector cannot contain metrics with the same labelset"), + "the rejection does not name the shared label set: {message}" ); // Two series carrying different hosts stay distinct, which is what a rate over a label that @@ -3948,18 +4467,271 @@ fn two_series_onto_one_label_set_after_a_window_are_rejected() { assert_eq!(out.len(), 2, "two series over two hosts are two series"); } +/// A name-dropping node over one metric is proved by the labels alone only when the scan's key +/// columns always decode a value; a scan whose keys can spell `NULL` and `''` is collected, and the +/// merge above the drop is what rejects the pair. +/// +/// `requests{host="web-1", zone=""}` and `requests{host="web-1"}` are two series to a scan and one +/// label set to everything that drops empty values: `timestamp(x)`, `rate(x[5m])` and +/// `timestamp(rate(x[5m]))` all drop the metric name and the empty `zone` with it. The name an +/// individual series carries is the one the whole scan carries, so the labels alone prove the drop +/// of that name - but not of the empty value: a primary-key column that holds `NULL` in one series +/// and `''` in another is exactly this pair, and the pass collects it when the scan states that its +/// keys cannot tell them apart. +/// +/// Without that fact the drop is proved and nothing is collected, which is what the last block pins: +/// the two hosts below are two series to the scan and two to every kernel above it. +/// +/// The statements collected for the key columns alone are counted, not silently paid for: see +/// `duplicate_labelset_check_skipped` in the query planner. +#[test] +fn a_name_dropping_node_over_a_nullable_key_is_collected() { + let out_grid = grid(0, 60_000, 5); + let schema = raw_schema(); + let counter = [0, 60_000, 120_000, 180_000, 240_000]; + let tags = vec![vec![ + "host".to_string(), + "zone".to_string(), + "__name__".to_string(), + ]]; + let metrics = [Some(Arc::::from("requests"))]; + // `requests{host=host}`, the shape the proved case is told apart by. + fn requests(host: &str) -> SeriesLabels { + SeriesLabels::try_new( + Some("requests".into()), + vec![Label { + name: "host".into(), + value: host.into(), + }], + ) + .unwrap() + } + // What the merge refuses, or `None` when every piece left it. + fn rejected(execution: &mut NativeExecutor) -> Option { + loop { + match execution.next_chunk() { + Ok(None) => return None, + // A piece may settle before a later one collides with it. + Ok(Some(_)) => {} + Err(error) => return Some(error.to_string()), + } + } + } + + let stamps = || { + let mut node = instant(input(), 300_000, 0, Arc::clone(&out_grid)); + let NativeExpr::Instant { op, .. } = &mut node.expr else { + unreachable!() + }; + op.value = InstantValue::Timestamp; + node + }; + let rate = || { + window_with( + input(), + RangeFunction::Rate, + 300_000, + None, + Arc::clone(&out_grid), + ) + }; + let evaluated_stamps = || { + let inner = rate(); + let output = inner.output.clone(); + NativeNode { + expr: NativeExpr::Timestamp { + input: Box::new(inner), + }, + output, + } + }; + let passed = |root: NativeNode, keys_survive: bool| { + let tree = NativeTree::try_new(root).unwrap(); + merge_label_rewrites(&tree, &tags, &metrics, keys_survive) + .expect("the pass produces a legal tree") + }; + let push = |execution: &mut NativeExecutor, second: SeriesLabels| { + execution + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-a", + host_metric(Some("")), + &counter, + &[1.0, 2.0, 3.0, 4.0, 5.0], + true, + ), + ) + .unwrap(); + execution + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-b", + second, + &counter, + &[6.0, 7.0, 8.0, 9.0, 10.0], + true, + ), + ) + .unwrap(); + }; + + // A key column that can hold `NULL` in one series and `''` in another: the drop is collected, + // and the merge is where the two pieces of one label set are refused. + let refused = |name: &str, root: NativeNode| { + let tree = passed(root, false); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "`{name}` over a nullable key is collected: {tree}" + ); + let mut execution = executor(&tree, pool()); + push(&mut execution, host_metric(None)); + execution.end_of_input().unwrap(); + let message = rejected(&mut execution) + .unwrap_or_else(|| panic!("`{name}` handed two pieces of one label set over")); + assert!( + message.contains("vector cannot contain metrics with the same labelset"), + "`{name}`: the rejection does not name the shared label set: {message}" + ); + }; + refused("timestamp", stamps()); + refused("rate", rate()); + refused("timestamp(rate)", evaluated_stamps()); + + // Two series a non-empty label tells apart stay two. Over keys that cannot spell the pair the + // drop is proved and nothing is collected - an ordinary `abs`/`rate`/`timestamp` over a normal + // metric table - and over a nullable key it is collected, which costs the merge but answers the + // same two series. + let kept = |name: &str, root: NativeNode| { + let tree = passed(root, true); + assert!( + !matches!(tree.root().expr, NativeExpr::Merge { .. }), + "`{name}` keeps every partition: {tree}" + ); + let mut execution = executor(&tree, pool()); + push(&mut execution, requests("web-2")); + let out = run(&mut execution); + assert_eq!(2, out.len(), "`{name}`: two series over two hosts are two"); + assert_ne!( + out[0].series.metric, out[1].series.metric, + "`{name}`: the two hosts were not two label sets" + ); + }; + kept("timestamp", stamps()); + kept("rate", rate()); + kept("timestamp(rate)", evaluated_stamps()); + let kept_over_a_nullable_key = |name: &str, root: NativeNode| { + let tree = passed(root, false); + assert!( + matches!(tree.root().expr, NativeExpr::Merge { .. }), + "`{name}` over a nullable key is collected: {tree}" + ); + let mut execution = executor(&tree, pool()); + push(&mut execution, requests("web-2")); + let out = run(&mut execution); + assert_eq!( + 2, + out.len(), + "`{name}`: two series over two hosts are two, collected or not" + ); + assert_ne!( + out[0].series.metric, out[1].series.metric, + "`{name}`: the two hosts were not two label sets" + ); + }; + kept_over_a_nullable_key("timestamp", stamps()); + kept_over_a_nullable_key("rate", rate()); + kept_over_a_nullable_key("timestamp(rate)", evaluated_stamps()); +} + +/// Two series a `timestamp` hands out under one label set are refused by the merge above it. +/// +/// `timestamp` reports a sample's own time under the metric name dropped, so `a{job="x"}` and +/// `b{job="x"}` are one label set the moment they leave it, whatever their input held. The merge +/// collecting its output is where the second value of one position is reported, as Prometheus +/// refuses two series of one label set; a single series passes through it unchanged. +#[test] +fn two_series_onto_one_label_set_after_a_timestamp_are_rejected() { + let grid = grid(0, 60_000, 2); + let schema = evaluated_schema(Arc::clone(&grid)); + let tree = || { + let input = evaluated_input(Arc::clone(&grid)); + let output = input.output.clone(); + NativeTree::try_new(merge(NativeNode { + expr: NativeExpr::Timestamp { + input: Box::new(input), + }, + output, + })) + .unwrap() + }; + + // One series: the merge groups the one label set `timestamp` leaves it and answers it. + let mut single = executor(&tree(), pool()); + single + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 0, + vec![Some(1.0), Some(2.0)], + true, + ), + ) + .unwrap(); + let out = run(&mut single); + assert_eq!(out.len(), 1, "one series is one answer"); + let mut expected = named_metric("a", "x"); + expected.remove("__name__"); + assert_eq!(out[0].series.metric, expected); + + // Two series: both leave `timestamp` under `{job="x"}` and both hold a value at position 0. + let mut execution = executor(&tree(), pool()); + for (name, values) in [ + ("a", vec![Some(1.0), Some(2.0)]), + ("b", vec![Some(3.0), Some(4.0)]), + ] { + execution + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), + ) + .unwrap(); + } + execution.end_of_input().unwrap(); + let error = match execution.next_chunk() { + Ok(None) => panic!("the timestamp of two series settled nothing at all"), + Ok(Some(chunk)) => panic!( + "a second series left `timestamp` under the first one's label set: {:?}", + decided(&[chunk]) + ), + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("vector cannot contain metrics with the same labelset") + && message.contains("job=\"x\""), + "the rejection does not name the label set the two now share: {message}" + ); +} + /// A chunk that holds a value where the same field already holds one is rejected. /// -/// Neither piece closes its series, and neither needs to: the refusal is about the positions -/// themselves. `v0` would carry two values at position 0 under one label set, which no merge can -/// resolve, so the node refuses rather than picking one. +/// Each piece is a whole source series that closes itself, and the refusal is about the positions +/// the two hold, not about the close: they are one label set to the merge, and both hold a value of +/// field 0 at position 0. `v0` would carry two values at position 0 under one label set, which no merge can +/// resolve, so the merge collecting the map's output refuses rather than picking one. #[test] fn two_series_holding_a_value_at_one_position_are_rejected() { let map_grid = grid(0, 60_000, 2); - let tree = NativeTree::try_new(map( + let tree = NativeTree::try_new(merge(map( evaluated_input(Arc::clone(&map_grid)), vec![MathOp::Abs], - )) + ))) .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); @@ -3971,17 +4743,12 @@ fn two_series_holding_a_value_at_one_position_are_rejected() { execution .push_evaluated( InputId::new(0), - evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, false), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), ) .unwrap(); } - // The first piece is fine on its own: nothing has occupied any position yet. - let first = execution - .next_chunk() - .unwrap() - .expect("the first piece is handed on as it arrives"); - assert_eq!(sample_values(&first), vec![Some(1.0), Some(2.0)]); - // The second one puts a second value of the same field at position 0. + // The merge holds both pieces until its input ends, and then reports the shared position. + execution.end_of_input().unwrap(); let error = match execution.next_chunk() { Ok(None) => panic!("the map produced no chunk at all"), Ok(Some(chunk)) => panic!( @@ -3991,9 +4758,11 @@ fn two_series_holding_a_value_at_one_position_are_rejected() { Err(error) => error, }; let message = error.to_string(); + assert!( + message.contains("vector cannot contain metrics with the same labelset"), + "{message}" + ); assert!(message.contains("job"), "{message}"); - assert!(message.contains("field 0"), "{message}"); - assert!(message.contains("position 0"), "{message}"); } /// Two closed series under one label set are two pieces of one logical series. @@ -4123,6 +4892,203 @@ fn merge_canonicalizes_complementary_series_and_flushes_only_at_eof() { assert!(execution.next_chunk().unwrap().is_none()); } +/// A table of `wide`- and `wider`-bucket histograms, so two samples of one field differ in payload. +/// +/// The payload is what a merge can lose and still answer the right positions, so every test below +/// names the histogram it expects rather than only counting rows. +fn two_sized_histograms() -> (Arc, u32, u32) { + let mut builder = HistogramBatchBuilder::new(); + let wide = push_histogram(&mut builder, 2); + let wider = push_histogram(&mut builder, 5); + (builder.finish().unwrap(), wide, wider) +} + +/// The histograms field 0 of `chunk` holds, in grid order: `(position, count, buckets)`. +fn histogram_rows(chunk: &EvaluatedChunk) -> Vec<(usize, f64, usize)> { + let field = chunk + .series + .fields + .get(0) + .expect("the chunk carries one field"); + chunk + .series + .time + .positions() + .map(|(position, row)| match field.sample(row) { + SampleRef::Histogram(view) => ( + position, + view.count, + view.positive.buckets.len() + view.negative.buckets.len(), + ), + SampleRef::Missing => panic!("position {position} of the answer holds no sample"), + SampleRef::Float(value) => { + panic!("position {position} of the answer is the float {value}") + } + }) + .collect() +} + +/// Native histogram frames merge by position across the canonical merge, payload and all. +/// +/// This is the merge a metadata drop over a nullable key pays for: one label set arrives as two +/// pieces from the same partition, each holding some of its positions, and what comes out is one +/// series with one histogram per position, each still carrying the payload it was pushed with. A +/// position neither piece holds stays a hole, and the merge owes nothing until its input ends - the +/// closing chunk it answers with is the merged series. +#[test] +fn merge_canonicalizes_complementary_histogram_pieces_and_flushes_only_at_eof() { + let grid = grid(0, 1, 4); + let tree = NativeTree::try_new(merge(evaluated_input(Arc::clone(&grid)))).unwrap(); + let schema = evaluated_schema(Arc::clone(&grid)); + let label = named_metric("same", "x"); + let (table, wide, wider) = two_sized_histograms(); + let mut execution = executor(&tree, pool()); + execution + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of( + &schema, + label.clone(), + 0, + vec![Some(wide), None, None], + &table, + true, + ), + ) + .unwrap(); + execution + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of(&schema, label, 1, vec![None, Some(wider)], &table, true), + ) + .unwrap(); + assert!( + execution.next_chunk().unwrap().is_none(), + "merge exposed output before EOF" + ); + execution.end_of_input().unwrap(); + let output = execution.next_chunk().unwrap().unwrap(); + assert!(output.is_last); + // Position 0 is the first piece's histogram, position 2 the second piece's, and position 1 - a + // hole in both - stays a hole rather than becoming a row. + assert_eq!( + vec![(0, 2.0, 2), (2, 5.0, 5)], + histogram_rows(&output), + "the merged histograms are not the ones the pieces held" + ); + assert!(execution.next_chunk().unwrap().is_none()); +} + +/// Two pieces of one label set holding a histogram at one position are refused. +/// +/// The merge collects the pieces of one label set into one series, so a position two pieces fill +/// would carry two values of one field. That is the collision Prometheus reports, and the histogram +/// payload changes nothing about it: the check reads whether a sample is there, not what it holds. +#[test] +fn a_histogram_merge_rejects_a_position_two_pieces_hold() { + let grid = grid(0, 1, 4); + let tree = NativeTree::try_new(merge(evaluated_input(Arc::clone(&grid)))).unwrap(); + let schema = evaluated_schema(Arc::clone(&grid)); + let label = named_metric("same", "x"); + let (table, wide, wider) = two_sized_histograms(); + let mut execution = executor(&tree, pool()); + for (start, index) in [(0, wide), (0, wider)] { + execution + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of( + &schema, + label.clone(), + start, + vec![None, Some(index)], + &table, + true, + ), + ) + .unwrap(); + } + execution.end_of_input().unwrap(); + let error = match execution.next_chunk() { + Ok(None) => panic!("the merge settled nothing at all"), + Ok(Some(chunk)) => panic!( + "a histogram position two pieces held was handed on: {:?}", + histogram_rows(&chunk) + ), + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("vector cannot contain metrics with the same labelset"), + "the rejection does not name the shared label set: {message}" + ); +} + +/// A float-producing histogram function above the canonical merge reduces what the merge collected. +/// +/// `histogram_count(rate(h[5m]))` is this shape: the window below it answers histograms, the drop it +/// makes over a nullable key is collected into a merge carrying those frames, and the map above the +/// merge answers one float per histogram it reads. The merge is what proves the frames crossed the +/// boundary intact - a count of a lost payload would not be the count the piece was pushed with - and +/// the map above it drops the positions holding nothing. +#[test] +fn a_float_producing_histogram_map_reduces_the_histograms_a_merge_collected() { + let grid = grid(0, 1, 4); + let tree = NativeTree::try_new(map_ops( + merge(evaluated_input(Arc::clone(&grid))), + vec![MapOp::Histogram(NativeHistogramFunction::Count)], + )) + .unwrap(); + let schema = evaluated_schema(Arc::clone(&grid)); + let label = named_metric("same", "x"); + let (table, wide, wider) = two_sized_histograms(); + let mut execution = executor(&tree, pool()); + execution + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of( + &schema, + label.clone(), + 0, + vec![Some(wide), None, None], + &table, + true, + ), + ) + .unwrap(); + execution + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of( + &schema, + label, + 1, + vec![None, Some(wider), None], + &table, + true, + ), + ) + .unwrap(); + // Both pieces close themselves, so the map's answer comes out of `run` without an explicit end + // of input; the merge still owes nothing until the second piece arrives. + let chunks = run(&mut execution); + let [chunk] = chunks.as_slice() else { + panic!("expected one chunk, got {}", chunks.len()); + }; + // The map answers one float per merged histogram, at the positions the merge kept, and the name + // the map drops is the one the pieces shared. + assert_eq!(chunk.series.metric.name(), None); + assert_eq!( + chunk + .series + .time + .positions() + .map(|(position, _)| position) + .collect::>(), + vec![0, 2] + ); + assert_eq!(sample_values(chunk), vec![Some(2.0), Some(5.0)]); +} + /// A parent reads a merge's answer only once the merge has flushed. /// /// `Merge` is the only node that answers nothing until its input ends, so a parent that treats it @@ -4554,21 +5520,16 @@ fn merge_rejects_overlapping_occupancy() { ); } -/// Two open chunks that share a label set are accepted where no position of a field collides. +/// Two open chunks that share a label set are handed on as they arrive. /// /// `a{job="x"}` and `b{job="x"}` are two series to the input and one label set to `abs`, which /// drops the name. Neither piece says it is the last chunk of its series, so the second one arrives /// while the first is still open. The two occupy disjoint positions of field 0 — a value at 0 and a -/// value at 3 — so the label-set check hands both on. -/// -/// The ordering is contiguous because that is the only ordering this runtime executes: the stream -/// below is not one a scan produces, since it never closes `a`, and what is checked here is the -/// occupancy the check reads rather than a delivery order. -/// -/// This is acceptance by that check, and it is not a merge: this layer passes every chunk through -/// as it arrives and keeps no sample state that would combine two series. Merging two series under -/// one label set is [`crate::grid_series::merge_same_labelset`]'s job, and nothing here performs -/// it. +/// value at 3 — so nothing about their labels is decided here: a map passes each chunk through as it +/// arrives, and combining two pieces of one label set is +/// [`crate::grid_series::merge_same_labelset`]'s job, which the merge a consumer reads them through +/// performs. A pair holding a value at the same position is what that merge refuses, not this +/// layer. #[test] fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides() { let map_grid = grid(0, 60_000, 4); @@ -4627,18 +5588,19 @@ fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides /// A `NaN` is a value and a missing sample is not, so only the `NaN` occupies its position. /// -/// `abs` drops the metric name, so the two series below are one label set to the map, which is where -/// the two are read against each other. A position holding no sample is a position the series never -/// claimed, and another series may fill it; a position holding a `NaN` is decided and holds a value, -/// like any other, so a second value there is the collision no merge resolves. +/// `abs` drops the metric name, so the two series below are one label set to the map, and the merge +/// above it is where the two are read against each other. A position holding no sample is a +/// position the series never claimed, and another series may fill it; a position holding a `NaN` is +/// decided and holds a value, like any other, so a second value there is the collision no merge +/// resolves. #[test] fn a_nan_occupies_its_position_where_a_missing_sample_does_not() { let map_grid = grid(0, 60_000, 1); let tree = || { - NativeTree::try_new(map( + NativeTree::try_new(merge(map( evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Contiguous), vec![MathOp::Abs], - )) + ))) .unwrap() }; let schema = evaluated_schema(Arc::clone(&map_grid)); @@ -4652,28 +5614,24 @@ fn a_nan_occupies_its_position_where_a_missing_sample_does_not() { (vec![Some(1.0)], vec![None], true), (vec![None], vec![Some(f64::NAN)], true), ]; - // Both chunks are open and neither needs to close: what the check reads is which positions a - // chunk occupies, and that does not depend on `is_last`. - let drain = |execution: &mut NativeExecutor| -> Result<(), DataFusionError> { - while execution.next_chunk()?.is_some() {} - Ok(()) - }; + // Each chunk is a whole source series that closes itself, so the merge reads the two as two + // pieces of one label set rather than as one series' runs. It holds both until the input ends + // and only then answers or refuses the pair. for (first, second, accepted) in cases { let mut execution = executor(&tree(), pool()); execution .push_evaluated( InputId::new(0), - evaluated_chunk_of(&schema, named_metric("a", "x"), 0, first.clone(), false), + evaluated_chunk_of(&schema, named_metric("a", "x"), 0, first.clone(), true), ) .unwrap(); - drain(&mut execution).expect("the first open chunk is handed on"); execution .push_evaluated( InputId::new(0), - evaluated_chunk_of(&schema, named_metric("b", "x"), 0, second.clone(), false), + evaluated_chunk_of(&schema, named_metric("b", "x"), 0, second.clone(), true), ) .unwrap(); - let outcome = drain(&mut execution); + let outcome = execution.finish().map(|_| ()); assert_eq!( outcome.is_ok(), accepted, @@ -4859,63 +5817,6 @@ fn a_window_retains_the_samples_of_every_field() { ); } -/// A chunk that produces nothing still leaves the table this node emitted into in its hands. -/// -/// The first chunk decided both grid points and the window handed them out, so the table holds a -/// label set and its runs. The second chunk decides nothing — every point was already decided — so -/// the window yields no chunk, but it has given none of that history back: the reservation has to -/// keep covering the table, not drop it. -#[test] -fn a_chunk_with_no_output_keeps_charging_the_emitted_table() { - let tree = NativeTree::try_new(window_with( - input(), - RangeFunction::LastOverTime, - 1, - None, - grid(0, 60_000, 2), - )) - .unwrap(); - let schema = raw_schema(); - let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool)); - - execution - .push_raw( - InputId::new(0), - raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false), - ) - .unwrap(); - let first = execution - .next_chunk() - .unwrap() - .expect("the first chunk decided both points"); - assert_eq!(sample_values(&first), vec![Some(1.0), Some(2.0)]); - drop(first); - let after_output = execution.state().reserved_bytes(); - assert!( - after_output > 0, - "the window handed out a position, which it has to hold" - ); - assert_eq!(after_output, pool.reserved()); - - // Both points are decided, so this chunk decides nothing and the window yields no chunk. - execution - .push_raw( - InputId::new(0), - raw_chunk(&schema, &[120_000], &[3.0], false), - ) - .unwrap(); - assert!( - execution.next_chunk().unwrap().is_none(), - "a chunk after every point was decided produced a chunk" - ); - assert!( - execution.state().reserved_bytes() > 0, - "a chunk with no output released the bytes the emitted table still holds" - ); - assert_eq!(execution.state().reserved_bytes(), pool.reserved()); -} - /// A window cannot run over an interleaved input, and the runtime says so before reading a chunk. /// /// The binding refuses a window whose own input declares the ordering, and the runtime refuses the @@ -5606,195 +6507,6 @@ fn dropping_a_node_drops_its_queue() { assert!(gauge(&metrics, "native.p0.source.peak_reserved_bytes") > 0); } -/// A chunk whose axis and fields disagree is refused, not read past the end of a field. -/// -/// [`Series`] keeps its fields public, so a chunk built by struct literal -- by a caller of this -/// crate as much as by a mistake here -- can carry fewer samples than the axis places rows. The -/// label-set ledger reads every row the axis names, so it is where such a chunk would be read out of -/// bounds; it reports the misalignment against the node that was handed it instead. -#[test] -fn a_series_whose_axis_and_fields_disagree_is_rejected() { - let grid = grid(0, 60_000, 4); - // Two samples placed on a four-row axis: the third row the axis names has nothing to read. - let samples = Samples::from_float(Float64Array::from(vec![1.0, 2.0])); - let time = TimeAxis::Grid(GridRange::try_new(Arc::clone(&grid), 0, 4).unwrap()); - let series = Series { - metric: named_metric("requests", "job"), - time, - fields: SeriesFields::try_from_vec(vec![samples]).unwrap(), - }; - let chunk = EvaluatedChunk::try_new(series, false).expect("the axis is addressed on a grid"); - - let mut emitted = Emitted::default(); - let error = emitted - .occupy(&chunk) - .expect_err("a field with fewer samples than the axis places rows was accepted"); - let message = error.to_string(); - assert!( - message.contains("misaligned") - && message.contains("4 rows") - && message.contains("2 samples"), - "the rejection does not name what the axis places against what the field holds: {message}" - ); -} - -/// A sparse series that only ever moves forward is appended to its history, not merged into it again -/// chunk after chunk. -/// -/// One legal series arrives as eight chunks of one position each, every position a hole apart from -/// the last: no label set is reached twice, no position of a field is occupied twice, and this is -/// what a scan produces as its positions advance. The stored runs still grow by one per chunk, so -/// rebuilding the array after each chunk moves `K(K+1)/2` runs to hold `K` rows of real data, which -/// is the cost this asserts away. -/// -/// The chunks go through [`super::Emitted::occupy`], the helper the runtime records an output with, -/// so what is checked is the history that helper keeps. Three shapes are pushed: chunks landing past -/// every stored run, which have to be appended to; a chunk starting where the last stored run ends, -/// which has to be joined to it rather than left beside it; and chunks landing before what is -/// stored, which cannot be appended to and have to go through the full merge while still leaving the -/// history sorted and disjoint. -#[test] -fn a_sparse_series_that_moves_forward_is_appended_to_its_history() { - let map_grid = grid(0, 60_000, 21); - let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut emitted = Emitted::default(); - - // One row at `position`, which is one run of one position: the same series every time, since a - // map dropping the name leaves them all one label set. - let push = |emitted: &mut Emitted, position: usize| { - let chunk = sparse_chunk( - &schema, - named_metric("a", "x"), - &[(position, 1)], - vec![-1.0], - false, - ); - emitted - .occupy(&chunk) - .unwrap_or_else(|error| panic!("position {position} was refused: {error}")); - }; - // The runs stored under the one label set, which is what `absorb` keeps in position order. - let stored = |emitted: &Emitted| -> Vec<(usize, usize)> { - emitted - .sets - .values() - .next() - .expect("the chunk was recorded under one label set") - .fields - .first() - .expect("the series has one field") - .iter() - .map(|run| (run.start, run.end)) - .collect() - }; - let ordered = |runs: &[(usize, usize)]| { - runs.iter().all(|(start, end)| start < end) - && runs.windows(2).all(|pair| pair[0].1 < pair[1].0) - }; - - // Eight chunks, one position each, a hole between them. - let positions = [4, 6, 8, 10, 12, 14, 16, 18]; - let before = absorb_rebuilds(); - for position in positions { - push(&mut emitted, position); - } - assert_eq!( - stored(&emitted), - vec![ - (4, 5), - (6, 7), - (8, 9), - (10, 11), - (12, 13), - (14, 15), - (16, 17), - (18, 19), - ], - "a sparse series was not stored as the one run each of its chunks occupies" - ); - assert_eq!( - absorb_rebuilds() - before, - 0, - "the {} chunks of a series moving forward rebuilt its history instead of appending to it", - positions.len() - ); - - // A chunk that starts where the last stored run ends touches it, and the two are one run: the - // fast path appends, and the boundary is the part it has to join. - let before = absorb_rebuilds(); - push(&mut emitted, 19); - assert_eq!( - stored(&emitted), - vec![ - (4, 5), - (6, 7), - (8, 9), - (10, 11), - (12, 13), - (14, 15), - (16, 17), - (18, 20), - ], - "a chunk touching the last stored run was left beside it rather than joined to it" - ); - assert_eq!( - absorb_rebuilds() - before, - 0, - "a chunk touching the last stored run was merged rather than appended to it" - ); - - // A chunk landing in a hole before the last stored run cannot be appended to: it joins the runs - // on both sides of it, so it goes through the full merge, which has to join them all the same. - let before = absorb_rebuilds(); - push(&mut emitted, 5); - assert_eq!( - stored(&emitted), - vec![ - (4, 7), - (8, 9), - (10, 11), - (12, 13), - (14, 15), - (16, 17), - (18, 20), - ], - "a chunk in a hole did not join the runs on either side of it" - ); - assert_eq!( - absorb_rebuilds() - before, - 1, - "a chunk landing before the last stored run was appended rather than merged" - ); - - // A chunk before every stored run is the other shape the fast path cannot take. - let before = absorb_rebuilds(); - push(&mut emitted, 1); - let runs = stored(&emitted); - assert_eq!( - runs, - vec![ - (1, 2), - (4, 7), - (8, 9), - (10, 11), - (12, 13), - (14, 15), - (16, 17), - (18, 20), - ], - "a chunk before every stored run did not land in front of them" - ); - assert_eq!( - absorb_rebuilds() - before, - 1, - "a chunk landing before every stored run was appended rather than merged" - ); - assert!( - ordered(&runs), - "the merged history is not sorted, disjoint and joined: {runs:?}" - ); -} - /// A bare selector stamps the last sample at or before each grid point onto its own grid. /// /// The chunk is addressed on the node's output grid, not on the source's timestamps; a grid point @@ -6110,9 +6822,9 @@ fn a_selected_histogram_keeps_its_start_timestamp() { /// Two series that differ only in an empty-valued label are two series, and both are answered. /// -/// The label set the runtime's occupancy table compares drops empty values, so a node that recorded -/// what it handed out under one would refuse the second series. A selection keeps the labels it -/// read, empty values included, so it keeps them apart and records nothing. +/// The label set a merge groups by drops empty values, so the two would be one set to a merge. A +/// selection keeps the labels it read, empty values included, so it hands the two on apart and +/// collects nothing. #[test] fn two_series_differing_only_in_an_empty_label_are_both_answered() { let out_grid = grid(0, 60_000, 1); @@ -10212,41 +10924,6 @@ fn name_of(chunk: &EvaluatedChunk) -> Option<&str> { chunk.series.metric.name() } -/// The [`Emitted`] tables a tree built from `node` keeps, as `(broadcast, window it reads)`. -/// -/// The second answer is `false` when the broadcast's vector side is not a window, which is what -/// tells a table its own node keeps from one the node below it already keeps. -fn emitted_tables(node: NativeNode) -> (bool, bool) { - let tree = NativeTree::try_new(node).unwrap(); - let mut inputs = Vec::new(); - let mut binary_before = 0; - let root = RuntimeNode::build( - tree.root(), - &pool(), - &mut inputs, - &mut binary_before, - "root".to_string(), - None, - ) - .unwrap(); - let RuntimeOp::Broadcast { - left, right, state, .. - } = root.op - else { - panic!("the tree did not build a broadcast at its root"); - }; - let vector = if matches!(left.op, RuntimeOp::Window { .. }) { - left - } else { - right - }; - let window = match vector.op { - RuntimeOp::Window { state, .. } => state.emitted.is_some(), - _ => false, - }; - (state.emitted.is_some(), window) -} - /// A constant on the right multiplies every value of the vector and drops the metric name. /// /// The chunk arrives as it is pushed, before the input ends: a broadcast answers a chunk from that @@ -11420,16 +12097,20 @@ fn a_broadcast_above_a_collecting_window_waits_for_its_flush() { } /// A broadcast rejects distinct input names that become one output label set at occupied points. +/// +/// The multiplication drops the metric name, so `requests` and `requests_total` hold one label set +/// above it and both hold a value at the same positions. The merge above the broadcast is what +/// refuses the pair: the node itself is point-wise and hands every chunk over as it arrives. #[test] fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reservation() { let grid = grid(0, 60_000, 2); let schema = evaluated_schema(Arc::clone(&grid)); - let tree = NativeTree::try_new(broadcast_right( + let tree = NativeTree::try_new(merge(broadcast_right( evaluated_input(Arc::clone(&grid)), 100.0, BinaryOp::Mul, false, - )) + ))) .unwrap(); let pool = pool(); let mut execution = executor(&tree, Arc::clone(&pool)); @@ -11441,15 +12122,6 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv evaluated_chunk_of(&schema, first_metric, 0, vec![Some(1.0), Some(2.0)], true), ) .unwrap(); - let first = execution - .next_chunk() - .unwrap() - .expect("first series output"); - assert_eq!( - decided(std::slice::from_ref(&first)), - vec![(0, Some(100.0)), (1, Some(200.0))] - ); - let second_metric = SeriesLabels::try_new(Some("requests_total".to_string().into()), Vec::new()).unwrap(); execution @@ -11458,6 +12130,8 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv evaluated_chunk_of(&schema, second_metric, 0, vec![Some(3.0), Some(4.0)], true), ) .unwrap(); + // The merge collects both mapped series and reports the collision when its input ends. + execution.end_of_input().unwrap(); let error = execution .next_chunk() .expect_err("the output label set and occupied positions conflict"); @@ -11466,8 +12140,7 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv // scalar broadcast produced no label set of its own, so the rendering is empty and the two // series are told apart only by what they produced. assert!( - message.contains("vector cannot contain metrics with the same labelset") - && message.contains("two series hold a value of field 0"), + message.contains("vector cannot contain metrics with the same labelset"), "expected the collision the node produces, got: {message:?}" ); assert_eq!( @@ -11482,84 +12155,6 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv ); } -/// A literal arithmetic broadcast hands out the label set its vector side already checked, so it -/// keeps no [`Emitted`] table of its own — and the window below it keeps its own. -/// -/// A table is what refuses a second series collapsing onto one label set, and the window is where a -/// dropped name makes that collapse happen. A second copy above it would record every label set the -/// window hands out to decide what the window has already decided. -#[test] -fn a_literal_arithmetic_over_a_name_dropping_window_keeps_no_emitted_table() { - let grid = grid(0, 60_000, 2); - let streaming = || { - window_with( - evaluated_input(Arc::clone(&grid)), - RangeFunction::Increase, - 120_000, - Some(Arc::clone(&grid)), - Arc::clone(&grid), - ) - }; - let collecting = || { - window_with( - map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]), - RangeFunction::Increase, - 120_000, - Some(Arc::clone(&grid)), - Arc::clone(&grid), - ) - }; - // `increase` drops the metric name, so both shapes stand on a window whose table already covers - // the identities this node would hand out -- and the operation is neither `rate` nor a - // multiplication, which is not what decides it. - assert_eq!( - emitted_tables(broadcast_right(streaming(), 2.0, BinaryOp::Add, false)), - (false, true), - "a streaming window repeats its table under a constant add" - ); - assert_eq!( - emitted_tables(broadcast_left(2.0, streaming(), BinaryOp::Add, false)), - (false, true), - "a constant on the left is the same one-to-one map" - ); - assert_eq!( - emitted_tables(broadcast_right(collecting(), 2.0, BinaryOp::Mul, false)), - (false, true), - "a collecting window repeats its table too" - ); - // A window that keeps the metric name hands out identities the operation has not seen: the - // multiplication drops the name, so two names of one tag set become one label set above it. - assert_eq!( - emitted_tables(broadcast_right( - window_with( - evaluated_input(Arc::clone(&grid)), - RangeFunction::LastOverTime, - 120_000, - Some(Arc::clone(&grid)), - Arc::clone(&grid), - ), - 2.0, - BinaryOp::Mul, - false, - )), - (true, true), - "a name-preserving window leaves the collision to the node above it" - ); - // The vector side is the source itself, so nothing below this node dropped a name and this node - // is the only one that can refuse: `a_broadcast_refuses_two_series_that_share_a_label_set_and_ - // releases_the_reservation` is the refusal that rests on the table it keeps. - assert_eq!( - emitted_tables(broadcast_right( - evaluated_input(Arc::clone(&grid)), - 2.0, - BinaryOp::Mul, - false, - )), - (true, false), - "a source under the broadcast leaves it the only node that can refuse" - ); -} - /// What a literal arithmetic broadcast hands on under a name-dropping window keeps its positions: /// series that differ only by name hold different ones, so nothing collides, each hands over its own /// close, and the constant reaches every value that is there. @@ -11727,23 +12322,24 @@ fn a_literal_arithmetic_over_a_collecting_window_hands_on_its_flush() { assert_eq!(pool.reserved(), 0); } -/// The window below a literal arithmetic broadcast refuses two series that collapse onto one label -/// set, and it does so before the aggregate above can fold them into one group. +/// The merge over a name-dropping window refuses two series that collapse onto one label set, and +/// it does so before the aggregate above can fold them into one group. /// -/// The broadcast keeps no table for this shape, so the refusal is the window's -- which is also what -/// keeps it upstream: a duplicate the partial aggregate read would land in one group and answer a -/// wrong value instead of refusing, and the reservations would be held until the fold answered. +/// `rate` drops the metric name, so the two series below leave the window under one label set: the +/// merge collecting the window's output is where the pair is refused. That placement is what keeps +/// the refusal upstream -- a duplicate the partial aggregate read would land in one group and answer +/// a wrong value instead of refusing, and the reservations would be held until the fold answered. #[test] -fn a_window_under_a_literal_arithmetic_refuses_the_collision_it_collapses() { +fn a_merge_over_a_name_dropping_window_refuses_before_the_aggregate() { let out_grid = grid(0, 60_000, 5); let tree = NativeTree::try_new(partial_sum(broadcast_right( - window_with( + merge(window_with( input(), RangeFunction::Rate, 300_000, None, Arc::clone(&out_grid), - ), + )), 2.0, BinaryOp::Mul, false, @@ -11771,8 +12367,8 @@ fn a_window_under_a_literal_arithmetic_refuses_the_collision_it_collapses() { execution.next_state().unwrap().is_none(), "the partial aggregate answered before its input ended" ); - // The second series is a different one to the scan and the same one to `rate`: the window is - // where the two meet, and the broadcast above it has no table to notice it. + // The second series is a different one to the scan and the same one to `rate`: the merge over + // the window is where the two meet, and the broadcast above it collects nothing of its own. execution .push_raw( InputId::new(0), @@ -11786,6 +12382,9 @@ fn a_window_under_a_literal_arithmetic_refuses_the_collision_it_collapses() { ), ) .unwrap(); + // The merge holds both series until its input ends, and then reports the pair it cannot file + // under one label set. + execution.end_of_input().unwrap(); let error = match execution.next_state() { Ok(None) => panic!("the second series settled nothing at all"), Ok(Some(_)) => panic!("a second series left the window under the first one's label set"), @@ -11794,15 +12393,11 @@ fn a_window_under_a_literal_arithmetic_refuses_the_collision_it_collapses() { let message = error.to_string(); assert!( message.contains("vector cannot contain metrics with the same labelset") - && message.contains("two series hold a value of field 0"), - "expected the window's refusal, got: {message:?}" - ); - // The node the refusal comes from is the one named: the window under the broadcast's left - // operand, not the broadcast that hands its answer on. - assert!( - message.contains("native.window at root.input.left"), - "the refusal does not name the window that made it: {message:?}" + && message.contains("host=\"web-1\""), + "expected the merge's refusal, got: {message:?}" ); + // The refusal is upstream of the fold: a duplicate it read would land in one group and answer a + // wrong value instead of refusing. assert_eq!( count(&metrics, "root", NodeMetric::OUTPUT_BATCHES), 0, @@ -11820,17 +12415,16 @@ fn a_window_under_a_literal_arithmetic_refuses_the_collision_it_collapses() { ); } -/// A window that keeps the metric name leaves the collision to the broadcast, which still keeps the -/// table that refuses it. +/// A window that keeps the metric name leaves the collision to the merge above the broadcast. /// /// `last_over_time` reports the name it read, so the two series stay distinct under it; the /// multiplication drops the name, and the two then hold one label set at the same positions. Nothing -/// below this node has checked that identity, so this node is where it is refused. +/// below the merge has collected that identity, so the merge above this node is where it is refused. #[test] -fn a_name_preserving_window_leaves_the_collision_to_the_broadcast() { +fn a_name_preserving_window_leaves_the_collision_to_the_merge_above_the_broadcast() { let grid = grid(0, 60_000, 2); let schema = evaluated_schema(Arc::clone(&grid)); - let tree = NativeTree::try_new(broadcast_right( + let tree = NativeTree::try_new(merge(broadcast_right( window_with( evaluated_input(Arc::clone(&grid)), RangeFunction::LastOverTime, @@ -11841,7 +12435,7 @@ fn a_name_preserving_window_leaves_the_collision_to_the_broadcast() { 2.0, BinaryOp::Mul, false, - )) + ))) .unwrap(); let pool = pool(); let mut execution = executor(&tree, Arc::clone(&pool)); @@ -11857,13 +12451,10 @@ fn a_name_preserving_window_leaves_the_collision_to_the_broadcast() { ), ) .unwrap(); - let first = execution - .next_chunk() - .unwrap() - .expect("the first series answers under its own name dropped"); - assert_eq!( - decided(std::slice::from_ref(&first)), - vec![(0, Some(2.0)), (1, Some(4.0))] + // The merge collects what the broadcast hands over, so nothing is answered yet. + assert!( + execution.next_chunk().unwrap().is_none(), + "the merge answered before its input ended" ); execution .push_evaluated( @@ -11877,14 +12468,14 @@ fn a_name_preserving_window_leaves_the_collision_to_the_broadcast() { ), ) .unwrap(); + execution.end_of_input().unwrap(); let error = execution .next_chunk() .expect_err("the two names become one label set at the positions both hold"); let message = error.to_string(); assert!( - message.contains("vector cannot contain metrics with the same labelset") - && message.contains("two series hold a value of field 0"), - "expected the collision the broadcast produces, got: {message:?}" + message.contains("vector cannot contain metrics with the same labelset"), + "expected the collision the merge produces, got: {message:?}" ); assert_eq!( pool.reserved(), @@ -11981,7 +12572,6 @@ fn a_histogram_window_under_literal_arithmetic_never_expands_occupancy() { } else { broadcast_right(window(), 2.0, op, false) }; - assert_eq!(emitted_tables(root.clone()), (false, true)); let tree = NativeTree::try_new(root).unwrap(); let pool = pool(); let collector = common_query::promql_annotations::PromqlAnnotationCollector::default(); @@ -13270,8 +13860,8 @@ fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() { for histogram in [false, true] { let grid = grid(1_000, 1_000, 2); let input = evaluated_input(Arc::clone(&grid)); - // Both sides are varying scalars: the broadcast retains one scalar series and emitted - // label history after producing the parameter. + // Both sides are varying scalars: the broadcast retains one scalar series after producing + // the parameter. let parameter = scalar_binary( eval_time(None, Arc::clone(&grid)), eval_time(None, Arc::clone(&grid)), @@ -13367,85 +13957,6 @@ fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() { } } -/// `scalar()` releases its own retained input after producing, but a map below it still holds -/// charged emitted-label history. Consuming the parameter must drop that subtree as well. -#[test] -fn dynamic_parameters_drop_scalar_of_map_history_before_terminal_close() { - for histogram in [false, true] { - let grid = grid(0, 1_000, 2); - let schema = evaluated_schema(Arc::clone(&grid)); - let input = evaluated_input(Arc::clone(&grid)); - let parameter = scalar_of(map( - evaluated_input_with_source_index(Arc::clone(&grid), 1), - vec![MathOp::Abs], - )); - let root = NativeNode { - output: input.output.clone(), - expr: if histogram { - NativeExpr::ParameterizedHistogram { - input: Box::new(input), - op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)), - parameters: vec![parameter], - } - } else { - NativeExpr::ParameterizedTopK { - input: Box::new(input), - op: BoundVectorOp::TopK { - k: 0.0, - top: true, - group: Grouping::By(Arc::from([])), - }, - parameter: Box::new(parameter), - } - }, - }; - let tree = NativeTree::try_new(root).unwrap(); - let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool)); - let metrics = metrics_of(&execution); - let parameter_name = if histogram { - "root.parameter0" - } else { - "root.parameter" - }; - let map_name = format!("{parameter_name}.input"); - execution - .push_evaluated( - InputId::new(1), - evaluated_chunk(&schema, 0, vec![1.0, 1.0], true), - ) - .unwrap(); - assert!(execution.next_chunk().unwrap().is_none()); - let map_current = format!("native.{map_name}.current_reserved_bytes"); - assert!( - gauge(&metrics, &map_current) > 0, - "map did not retain emitted history" - ); - // Drive exactly the finishing stage that produces the scalar, before the executor closes. - assert!(execution.live_mut().unwrap().pull(true).unwrap().is_none()); - assert_eq!(gauge(&metrics, &map_current), 0); - assert!( - gauge( - &metrics, - &format!("native.p0.{map_name}.peak_reserved_bytes") - ) > 0 - ); - assert_eq!(count(&metrics, &map_name, NodeMetric::OUTPUT_BATCHES), 1); - let root = &execution.state().live.as_ref().unwrap().root; - match &root.op { - RuntimeOp::ParameterizedTopK { parameter, .. } => assert!(parameter.is_none()), - RuntimeOp::ParameterizedHistogram { parameters, .. } => { - assert!(parameters[0].is_none()) - } - _ => panic!("expected dynamic operator"), - } - assert_eq!(execution.state().reserved_bytes(), pool.reserved()); - execution.live_mut().unwrap().root.finish_windows().unwrap(); - drop(execution); - assert_eq!(pool.reserved(), 0); - } -} - /// A settled parameter with an open window must report the window's finish error before dropping it. #[test] fn dynamic_parameters_finish_windows_before_dropping_the_subtree() { @@ -14816,17 +15327,51 @@ fn count_values_reserves_a_group_per_field_sample() { /// `label_replace` runs in a map: each series is relabelled as it passes, and two series it gives /// one label set collide where both hold a value, as Prometheus' same-labelset check refuses. +/// +/// The map rewrites a label its input carries, so the canonical pass collects what it hands out: the +/// merge above the rewrite is where the two pieces are read against each other. #[test] fn a_label_rewrite_relabels_series_and_refuses_a_shared_point() { let grid = grid(0, 60_000, 2); let schema = evaluated_schema(Arc::clone(&grid)); let rewrite = LabelRewrite::replace("job", "all", "job", ".*").unwrap(); - let tree = NativeTree::try_new(map_ops( - evaluated_input(Arc::clone(&grid)), - vec![MapOp::Label(Arc::new(rewrite))], - )) - .unwrap(); - let mut execution = executor(&tree, pool()); + let tree = |merge_above: bool| { + let rewritten = map_ops( + evaluated_input(Arc::clone(&grid)), + vec![MapOp::Label(Arc::new( + LabelRewrite::replace("job", "all", "job", ".*").unwrap(), + ))], + ); + NativeTree::try_new(if merge_above { + merge(rewritten) + } else { + rewritten + }) + .unwrap() + }; + assert_eq!(rewrite.destination(), "job"); + + // The rewrite itself: one series passes and leaves under the label it wrote. + let mut relabelled = executor(&tree(false), pool()); + relabelled + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 0, + vec![Some(1.0), Some(2.0)], + true, + ), + ) + .unwrap(); + let first = relabelled + .next_chunk() + .unwrap() + .expect("the relabelled series"); + assert_eq!(first.series.metric, named_metric("a", "all")); + + let mut execution = executor(&tree(true), pool()); execution .push_evaluated( InputId::new(0), @@ -14839,21 +15384,17 @@ fn a_label_rewrite_relabels_series_and_refuses_a_shared_point() { ), ) .unwrap(); - let first = execution - .next_chunk() - .unwrap() - .expect("the relabelled series"); - assert_eq!(first.series.metric, named_metric("a", "all")); execution .push_evaluated( InputId::new(0), evaluated_chunk_of(&schema, named_metric("a", "y"), 1, vec![Some(3.0)], true), ) .unwrap(); + execution.end_of_input().unwrap(); let error = execution.next_chunk().unwrap_err().to_string(); assert!( error.contains("vector cannot contain metrics with the same labelset") - && error.contains("two series hold a value of field 0 at position 1"), + && error.contains("job=\"all\""), "{error}" ); } diff --git a/src/promql/src/native/mod.rs b/src/promql/src/native/mod.rs index 3f561281e3d..0012c28b03d 100644 --- a/src/promql/src/native/mod.rs +++ b/src/promql/src/native/mod.rs @@ -34,7 +34,6 @@ pub mod display; pub mod execute; pub mod metrics; pub mod plan; -pub(crate) mod runs; pub mod segment; pub(crate) mod split; pub(crate) mod state; diff --git a/src/promql/src/native/plan.rs b/src/promql/src/native/plan.rs index f30a7e67d5f..7de7de907fe 100644 --- a/src/promql/src/native/plan.rs +++ b/src/promql/src/native/plan.rs @@ -1307,9 +1307,10 @@ impl NativeNode { /// A canonical label set is the identity a [`NativeExpr::Merge`] groups by: the metric name plus the /// ordinary labels, which is exactly what [`SeriesLabels`] holds. One scan partition holds one /// series per such set - a stored series is one identity, and the scan splits rows by storage key -/// rather than by label set - so only a computation that *writes* an existing label can turn two of -/// those identities into one. A `label_replace` or `label_join` does exactly that, and it may name -/// the very label that told its input's series apart. +/// rather than by label set - so only a computation that *writes* an existing label, or drops one, +/// can turn two of those identities into one. A `label_replace` or a `label_join` writes one, and it +/// may name the very label that told its input's series apart; a range function that drops the +/// metric name, and `timestamp` in either of its forms, drop one. /// /// The rewrite is judged by the labels it writes and the ones its input carries, which is why the /// caller supplies what it knows about each source's tags, indexed by the `source_index` the leaves @@ -1331,23 +1332,47 @@ impl NativeNode { /// /// Everything else either keeps the set it read or answers a set of its own: /// -/// - A map of math drops the metric name, and an arithmetic broadcast does too, but within one -/// metric the remaining tags are the primary key: two stored series of one metric that agree on -/// every other label are the same series, so dropping the name cannot make them one. Each of -/// those nodes hands its input's answer on rather than claiming one of its own. +/// - A map of math drops the metadata labels, and so do a histogram map, a parameterized map, an +/// arithmetic or `bool` scalar broadcast, a range function that computes its result and a +/// `timestamp` in either of its forms. Such a node is judged by two facts +/// ([`metadata_drop_keeps_series_apart`]): the labels it and its input report, and the scan's own +/// key columns. A source bound to one metric names every series it hands over, so the name the +/// drop removes told none of them apart, and a source that may hand over several is collected; and +/// because the drop removes every empty label value with the name, a key column that holds `NULL` +/// in one series and `''` in another - an absent and an empty value, which is one label set once +/// the empty one is gone - is collected too. `keys_survive` is that second fact, stated by the +/// caller for the scan below the tree +/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), which is +/// the same rule the physical per-series scan applies); a caller that cannot state it passes +/// `false`, and the conservative answer is the one that collects. /// - An `Aggregate` answers its groups, a `Vector` that folds series answers its result sets, and a /// `Binary` that reads whole vectors files its rows by the label set a sample *ends* under and /// refuses two values of a field at one point. A `Set` does the same, and `info` merges the /// results that share a label set. Each answers one series per set it names. -/// - A node that hands its input's series over unchanged - a selection, a window, a timestamp, a -/// parameterized map, `topk` and its ranking siblings - passes the answer on: an alias below it is -/// still an alias above it. +/// - A `Window` that drops the metadata labels, and the two timestamps, claim one of their own: the +/// name is part of the identity a merge groups by, so two series that shared every other label are +/// one label set from there on - unless the drop tells them apart by the same two-fact rule the +/// kernels above are judged by. +/// - The empty-value half of that proof is answered by a [`Merge`](NativeExpr::Merge) at or below the +/// drop ([`canonical_below`]): a merge hands over one series per canonical label set - the +/// comparison that treats an empty label value as no label - so no drop above it can fold two of +/// its series by removing an empty value. What a merge does not answer is the metadata labels +/// themselves, which the labels half below still has to prove. That is what keeps a chain of drops +/// over one metric to the one collection at its bottom rather than one per drop. +/// - A node that hands its input's series over unchanged - a selection, a name-keeping window, +/// `topk` and its ranking siblings - passes the answer on: an alias below it is still an alias +/// above it. /// /// This is the fact [`merge_label_rewrites`](crate::native::canonicalize::merge_label_rewrites) /// inserts a `Merge` for: a consumer that reads a whole label set cannot answer from pieces, and this /// says where pieces of one set can exist at all. A `Merge` is the collection itself, so it answers /// `false` - which is what makes the pass idempotent. -pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec]) -> bool { +pub fn duplicates_canonical_labels( + node: &NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> bool { match &node.expr { NativeExpr::Input(_) | NativeExpr::Scalar(_) @@ -1355,13 +1380,50 @@ pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec | NativeExpr::VectorOf { .. } => false, // The collection itself: one series per label set comes out of it. NativeExpr::Merge { .. } => false, - // These hand each series over as they read it, so what aliases below them still aliases. - NativeExpr::Instant { input, .. } - | NativeExpr::Window { input, .. } - | NativeExpr::Timestamp { input } - | NativeExpr::ParameterizedMap { input, .. } - | NativeExpr::ParameterizedTopK { input, .. } => { - duplicates_canonical_labels(input, source_tags) + // These hand each series over as they read it, so what aliases below them still aliases -- + // except `timestamp`, which reports a sample's own time under the metadata labels dropped + // and is therefore a source of duplicates in its own right. A merge below it already hands + // over one series per label set, so nothing the drop can fold is left there. + NativeExpr::Instant { input, op } => { + (op.value == InstantValue::Timestamp + && !metadata_drop_keeps_series_apart( + input, + source_tags, + source_metrics, + keys_survive, + )) + || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + } + // A window that drops the metadata labels can fold two series that agreed on every other + // label into one set, so it is a source of duplicates whether or not its input was one - + // unless the labels and the scan's key columns provably tell those series apart still, or a + // merge below it already collected them. + NativeExpr::Window { input, op } => { + (op.function.drops_metric_name() + && !metadata_drop_keeps_series_apart( + input, + source_tags, + source_metrics, + keys_survive, + )) + || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + } + // `timestamp` always drops the metadata labels, so two series it reads can become one set - + // unless the labels below it are all bound to one metric and the scan's keys survive the + // drop. + NativeExpr::Timestamp { input } => { + !metadata_drop_keeps_series_apart(input, source_tags, source_metrics, keys_survive) + || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + } + // A parameterized map is a math kernel - `round`, `clamp` - and drops the metadata labels as + // a math map does, so it answers for the same reason. Its parameter is a scalar and carries + // no labels of the input's. + NativeExpr::ParameterizedMap { input, .. } => { + !metadata_drop_keeps_series_apart(input, source_tags, source_metrics, keys_survive) + || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + } + NativeExpr::ParameterizedTopK { input, .. } => { + duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) } // `scalar()` answers one value, which names no label set at all. NativeExpr::ScalarOf { .. } => false, @@ -1373,9 +1435,21 @@ pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec // leaves every series' identity distinct. `map_writes_existing_label` is that question, and // an input whose labels this walk cannot derive is collected rather than trusted. Even a // whole-vector consumer can emit distinct series whose labels a later rewrite makes equal. + // A math or histogram kernel in the same map drops the metadata labels, which is a source of + // duplicates of its own unless the labels and the scan's keys tell its input's series apart - + // see the doc comment above. NativeExpr::Map { input, op } => { - map_writes_existing_label(input, op, source_tags).unwrap_or(true) - || duplicates_canonical_labels(input, source_tags) + (op.ops + .iter() + .any(|operation| matches!(operation, MapOp::Math(_) | MapOp::Histogram(_))) + && !metadata_drop_keeps_series_apart( + input, + source_tags, + source_metrics, + keys_survive, + )) + || map_writes_existing_label(input, op, source_tags).unwrap_or(true) + || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) } // A fold answers one series per group, which is one label set per group. NativeExpr::Aggregate { .. } => false, @@ -1384,23 +1458,49 @@ pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec BoundVectorOp::TopK { .. } | BoundVectorOp::LimitK { .. } | BoundVectorOp::LimitRatio { .. } - | BoundVectorOp::Sort(_) => duplicates_canonical_labels(input, source_tags), + | BoundVectorOp::Sort(_) => { + duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + } // `absent` reports one series, `count_values` one per distinct value, and the histogram // functions fold the bucket series of one histogram into one series. BoundVectorOp::Absent { .. } | BoundVectorOp::Histogram(_) | BoundVectorOp::CountValues { .. } => false, }, - NativeExpr::Binary { left, right, .. } => { + NativeExpr::Binary { + left, + right, + matching, + } => { let vector = |side: &NativeNode| side.output.samples().is_some(); - if vector(left) && vector(right) { + match (vector(left), vector(right)) { // Two vectors are matched by label, and the kernel files its rows by the label set a // sample ends under, refusing two values of one field at one point. - false - } else { - // A broadcast reaches each series of its vector side as it is. - let carried = if vector(left) { left } else { right }; - duplicates_canonical_labels(carried, source_tags) + (true, true) => false, + // A broadcast reaches each series of its vector side as it is. An arithmetic one and + // one that reports a `bool` drop the metadata labels, which is a source of + // duplicates of its own unless the labels and the scan's keys tell its series apart; + // a filtering comparison keeps the set it read. A merge below the vector side already + // handed over one series per label set, so the drop has nothing left to fold there. + (true, false) | (false, true) => { + let carried = if vector(left) { left } else { right }; + let drops_metadata = !matching.op.is_comparison() || matching.return_bool; + (drops_metadata + && !metadata_drop_keeps_series_apart( + carried, + source_tags, + source_metrics, + keys_survive, + )) + || duplicates_canonical_labels( + carried, + source_tags, + source_metrics, + keys_survive, + ) + } + // Two scalars name no label set at all. + (false, false) => false, } } // A set operation keeps each side's samples and files them by result label set, refusing two @@ -1447,6 +1547,313 @@ fn map_writes_existing_label( } Some(existing) } + +/// Whether the metadata labels a rewrite drops are the only ones telling `node`'s series apart. +/// +/// A range function that computes its result and `timestamp` in either of its forms drop +/// [`PROMQL_METADATA_LABELS`] - the metric name, `__type__` and `__unit__` - and a consumer above +/// them that reads a whole label set has to establish those series cannot collide once the three are +/// gone. Two series of one input collide only if they differed in one of the three alone, +/// so the drop is injective - and nothing has to be collected - when neither can tell two of them +/// apart: +/// +/// - The name: a source bound to one metric names every series it hands over, so the name is the +/// same for all of them and removing it merges nothing (see [`single_metric_name`]). A source that +/// may hand over several - one selector that discovered several metrics, or a caller that cannot +/// tell - is not proved, and the conservative answer is the one that collects. +/// - `__type__` and `__unit__`: both are removed along with the name, and neither is a label this +/// walk can prove constant across a source's series. They are ordinary labels when a source +/// carries them as tags or a rewrite writes them, and two series that agree on everything else are +/// one label set once they are gone. Their presence anywhere in `node`'s output is therefore +/// enough to withhold the proof. +/// +/// A node whose labels already lost the name - `abs`, a name-dropping window below it - has no name +/// of its own left to remove, and what its surviving labels tell apart is then only as good as the +/// metrics below it: a drop that happened on the way there removed a name without asking this walk, +/// so the proof asks instead that every source it reads is bound to the same one metric +/// ([`single_source_metric`]). Over a selector that discovered several metrics that drop folds them +/// together, and the answer is the one that collects. +/// +/// `false` when the walk cannot derive `node`'s labels at all, because an unknown set proves nothing. +/// +/// This is the whole of the drop's proof: every arm of [`duplicates_canonical_labels`] that decides +/// whether a collection is needed asks it, and so does [`unifies_its_own_contributions`] at run +/// time. What it cannot see is a key column that holds `NULL` in one series and `''` in another, +/// both of which decode to no label once the empty value is dropped with the name. That fact is not +/// derivable from a tree - it is read from the resolved table - so the caller states it as +/// `keys_survive`, using the same rule the physical per-series scan applies +/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop)). +/// [`leaves_a_drop_to_the_storage_keys`] is what makes the statements collected for that reason +/// countable. +fn metadata_drop_keeps_series_apart( + node: &NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> bool { + // The empty-value half is asked first, because it is the cheap one and because a merge at or + // below `node` answers it for the tree ([`canonical_below`]): a merge hands over one series per + // canonical label set - the comparison that treats an empty label value as no label - so two of + // its series are already distinct under it, and no drop above can fold them by removing an empty + // value. What a merge does *not* answer is the metadata labels themselves: it keeps two series + // that differ in the name, `__type__` or `__unit__`, and those are exactly what the drop + // removes, so the labels half still has to hold on its own. + // + // A caller that cannot state the key half - one holding no table, or a tree whose scans + // disagree - passes `keys_survive = false`, and only a merge below it can then carry the drop. + (keys_survive || canonical_below(node)) + && metadata_labels_keep_series_apart(node, source_tags, source_metrics) +} + +/// Whether the labels alone tell two of `node`'s series apart once the metadata labels are dropped. +/// +/// This is the half of [`metadata_drop_keeps_series_apart`] a tree can answer about itself; the other +/// half is the empty values the drop removes - the scan's key columns, or a merge that has already +/// told those apart. See that function's doc comment for the rule itself. +fn metadata_labels_keep_series_apart( + node: &NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], +) -> bool { + let Some(tags) = fused_output_tags(node, source_tags) else { + return false; + }; + if tags + .iter() + .any(|tag| tag == "__type__" || tag == "__unit__") + { + return false; + } + if !tags.iter().any(|tag| tag == METRIC_NAME) { + // No name is left to lose, so what the surviving labels tell apart is decided by the sources + // below: every one of them reading the same single metric is what says a name dropped on the + // way here could not have folded two metrics together either. `abs` and a name-dropping + // window both drop the name without asking, so without that fact nothing below is proved. + return single_source_metric(node, source_metrics).is_some(); + } + single_metric_name(node, source_metrics).is_some() +} + +/// Whether a [`NativeExpr::Merge`] collects the series `node` hands over, at it or below it. +/// +/// A merge holds one series per canonical label set, so the series above one are not stored series +/// any more: a merge with nothing to merge hands its input's pieces through, and one that does +/// merge keeps the first piece's labels and drops the pieces it folded, so what comes out is unique +/// under the label set Prometheus compares (`label_set` in `grid_series`) - the one that treats an +/// empty label value as no label. Two pieces that came out of one merge are therefore distinct under +/// that comparison even though their rows may still spell an empty value, which is what makes a +/// second drop above the merge unable to fold them by removing an empty value. What it does not make +/// impossible is the rest of such a drop: a merge keeps two series that differ in `__name__`, +/// `__type__` or `__unit__`, and removing those is exactly what a metadata drop does. +/// +/// Only the children that carry samples are searched: a parameter is a scalar or a vector of +/// constants, and a merge below one reads series this node's own samples did not come from. +fn canonical_below(node: &NativeNode) -> bool { + matches!(node.expr, NativeExpr::Merge { .. }) + || node + .expr + .children() + .into_iter() + .filter(|child| matches!(child.output, NativeType::Samples(_))) + .any(canonical_below) +} + +/// Whether `node` drops [`PROMQL_METADATA_LABELS`] on its own over series whose labels are proved to +/// tell apart and whose keys do not. +/// +/// This is the question of what the drop costs: the labels half of the proof +/// ([`metadata_labels_keep_series_apart`]) holds, nothing at or below the drop has collected already +/// ([`canonical_below`]), and the only reason a collection is still needed is the scan's key columns, +/// a primary-key column that holds `NULL` in one series and `''` in another. Those two decode to an +/// absent and to an empty label, and the drop normalizes them into one. Such a statement is now +/// collected for that reason alone ([`metadata_drop_keeps_series_apart`]), and this is the count +/// that says so. +/// +/// Every node that drops the metadata labels on its own counts: a map of math, a histogram map, a +/// parameterized map and an arithmetic or `bool` scalar broadcast, and the three that rewrite a +/// sample's own value - a name-dropping range function, `timestamp(x)` and a `timestamp` over an +/// evaluated input. +fn drops_metadata_labels_over_one_metric( + node: &NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> bool { + // The statement is counted exactly when the drop is collected for the key columns alone: the + // labels prove it ([`metadata_labels_keep_series_apart`]), nothing at or below it has collected + // already ([`canonical_below`]), and the keys do not survive it - which is the one fact the pass + // read from the resolved table and the reason the merge is there. + let decided_by_the_keys = |carried: &NativeNode| { + !keys_survive + && metadata_labels_keep_series_apart(carried, source_tags, source_metrics) + && !canonical_below(carried) + }; + let dropped = match &node.expr { + NativeExpr::Instant { input, op } => { + op.value == InstantValue::Timestamp && decided_by_the_keys(input) + } + NativeExpr::Timestamp { input } => decided_by_the_keys(input), + NativeExpr::Window { input, op } => { + op.function.drops_metric_name() && decided_by_the_keys(input) + } + NativeExpr::ParameterizedMap { input, .. } => decided_by_the_keys(input), + NativeExpr::Map { input, op } => { + op.ops + .iter() + .any(|operation| matches!(operation, MapOp::Math(_) | MapOp::Histogram(_))) + && decided_by_the_keys(input) + } + // A broadcast reaches each series of its vector side as it is; an arithmetic one and one + // that reports a `bool` drop the metadata labels, a filtering comparison keeps them. + NativeExpr::Binary { + left, + right, + matching, + } => { + let carried = match ( + left.output.samples().is_some(), + right.output.samples().is_some(), + ) { + (true, false) => Some(left.as_ref()), + (false, true) => Some(right.as_ref()), + _ => None, + }; + carried.is_some_and(|carried| { + (!matching.op.is_comparison() || matching.return_bool) + && decided_by_the_keys(carried) + }) + } + _ => false, + }; + dropped + || node.expr.children().into_iter().any(|child| { + drops_metadata_labels_over_one_metric(child, source_tags, source_metrics, keys_survive) + }) +} + +/// Whether `tree` holds a node that drops the metadata labels over series whose labels are proved to +/// tell apart and whose keys do not (see [`drops_metadata_labels_over_one_metric`]). +/// +/// A statement this answers `true` for is one the canonical pass collects for the key columns alone: +/// the labels prove the drop injective, nothing below it has collected already, and the scan below it +/// keys its series by a column that can hold `NULL` in one series and `''` in another +/// (`keys_survive` is `false`, and the caller states that fact from the resolved table). Those +/// statements are the price of reading that fact instead of the label-set check the physical scan +/// only states for itself, so the caller counts them by this answer. +pub fn leaves_a_drop_to_the_storage_keys( + tree: &NativeTree, + source_tags: &[Vec], + source_metrics: &[Option>], + keys_survive: bool, +) -> bool { + drops_metadata_labels_over_one_metric(tree.root(), source_tags, source_metrics, keys_survive) +} + +/// The one metric every source `node` reads is bound to, or `None` when they are not all one. +/// +/// This is the facts-free question about the leaves alone: a node can drop the metric name on the +/// way (a math map, a range function) whatever its sources hold, so what the surviving labels tell +/// apart is only as good as the set of metrics below them. +fn single_source_metric( + node: &NativeNode, + source_metrics: &[Option>], +) -> Option> { + let mut pending = vec![node]; + let mut metric: Option<&Arc> = None; + while let Some(node) = pending.pop() { + match &node.expr { + // A source that may hand over several metrics proves nothing about their names. + NativeExpr::Input(input) => { + let bound = source_metrics.get(input.source_index)?.as_ref()?; + match metric { + None => metric = Some(bound), + Some(seen) if seen == bound => {} + Some(_) => return None, + } + } + NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => {} + NativeExpr::Instant { input, .. } + | NativeExpr::Window { input, .. } + | NativeExpr::Map { input, .. } + | NativeExpr::Timestamp { input } + | NativeExpr::VectorOf { input } + | NativeExpr::Merge { input } + | NativeExpr::Aggregate { input, .. } + | NativeExpr::Vector { input, .. } + | NativeExpr::ScalarOf { input } => pending.push(input), + NativeExpr::ParameterizedMap { + input, parameters, .. + } + | NativeExpr::ParameterizedHistogram { + input, parameters, .. + } => { + pending.push(input); + pending.extend(parameters.iter()); + } + NativeExpr::ParameterizedTopK { + input, parameter, .. + } => { + pending.push(input); + pending.push(parameter); + } + NativeExpr::Binary { left, right, .. } + | NativeExpr::Set { left, right, .. } + | NativeExpr::Info { + base: left, + info: right, + .. + } => { + pending.push(left); + pending.push(right); + } + } + } + metric.cloned() +} + +/// The one metric every series `node` hands over is named, or `None` when this walk cannot say. +/// +/// Only the nodes that hand their input's label set on unchanged can carry a name: every kernel that +/// reports a value of its own drops it, so a name above one of those is one a rewrite wrote. What is +/// left is the boundary the leaves are bound to - where `source_metrics` answers, indexed by the +/// `source_index` the leaf reads - and the nodes that pass a set through. +fn single_metric_name(node: &NativeNode, source_metrics: &[Option>]) -> Option> { + match &node.expr { + NativeExpr::Input(input) => source_metrics.get(input.source_index).cloned().flatten(), + // These report the label set they read rather than one of their own, so the name is their + // input's. A map that computes a value or writes a name is not one of them. + NativeExpr::Merge { input } + | NativeExpr::ParameterizedTopK { input, .. } + | NativeExpr::Vector { + input, + op: + BoundVectorOp::TopK { .. } + | BoundVectorOp::LimitK { .. } + | BoundVectorOp::LimitRatio { .. } + | BoundVectorOp::Sort(_), + } => single_metric_name(input, source_metrics), + NativeExpr::Instant { input, op } if op.value != InstantValue::Timestamp => { + single_metric_name(input, source_metrics) + } + NativeExpr::Window { input, op } if !op.function.drops_metric_name() => { + single_metric_name(input, source_metrics) + } + NativeExpr::Map { input, op } + if op.ops.iter().all(|operation| { + matches!( + operation, + MapOp::Label(rewrite) if rewrite.destination() != METRIC_NAME + ) + }) => + { + single_metric_name(input, source_metrics) + } + // A kernel that reports a value of its own drops the name, and a fold, a set operation, an + // `info` lookup and a rewrite writing the name answer a set this walk does not follow. + _ => None, + } +} + /// Whether `node` reads the whole input vector at a point rather than one series at a time. /// /// An `Aggregate` folds every series of its group, a `Binary` and a `Set` pair two vectors' series @@ -1464,11 +1871,16 @@ fn map_writes_existing_label( /// Both a region-side placement and the native route's admission ask this, and they must agree: a /// tree admitted here but refused there becomes a node the physical layer cannot build, and one /// refused here but admitted there would be pushed to a region that holds part of the metric. +/// +/// The walk asks the facts-free form of [`unifies_its_own_contributions`]: a placement reads a tree +/// without the binding that says which metric each of its inputs reads, and every reader of this +/// answer - the admission below, the region-side classification and the locality of a built node - +/// asks it of the same tree, so the conservative answer is the one that keeps them agreeing. pub fn reads_across_series(node: &NativeNode) -> bool { match &node.expr { NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => false, NativeExpr::Window { input, .. } => { - matches!(unifies_its_own_contributions(input), Some(false)) + matches!(unifies_its_own_contributions(input, &[], &[]), Some(false)) || reads_across_series(input) } NativeExpr::Instant { input, .. } @@ -2492,8 +2904,37 @@ fn broadcast_drops_labels(matching: &BinaryMatching) -> bool { /// /// The collection condition is `matches!(answer, Some(false))`: the Window `MergeInput` collector /// at runtime and the `limitk` collector both ask this predicate of their input. +/// +/// `source_tags` and `source_metrics` are the same facts +/// [`duplicates_canonical_labels`] judges a whole label set by, indexed the same way: a node that +/// drops the metadata labels reports fragments only when +/// [`metadata_labels_keep_series_apart`] cannot prove the labels tell its series apart. That is the +/// labels half of the drop's proof, and it is the half a tree can answer about itself. The other half +/// is whether the scan's key columns can spell one series as `NULL` and another as `''`, both of +/// which the drop normalizes into one label set: it is stated for the tree as a whole and is asked by +/// the canonical pass ([`duplicates_canonical_labels`], which reads it as +/// [`metadata_drop_keeps_series_apart`]). Where that half does not hold, the pass has put its merge +/// above this node already, so this predicate is asked of the merge instead, which is why asking the +/// labels half here cannot disagree with the pass about a drop. The statements that half collects +/// are counted by [`leaves_a_drop_to_the_storage_keys`]. +/// +/// This predicate is only ever asked of a node by the consumer directly above it - the `Window` +/// `MergeInput` collector and the `limitk` one, both in [`execute`](crate::native::execute) - and +/// that is exactly where the canonical pass puts its [`NativeExpr::Merge`] when a collection is +/// needed: it collects the input of a `Window`, an `Aggregate`, a `Vector`, a `Map`, a +/// `ParameterizedMap` and each side of a `Binary`/`Set`/`Info` +/// ([`merge_label_rewrites`](crate::native::canonicalize::merge_label_rewrites)). The collector +/// then reads the merge's `Some(true)` - a merge *is* the collection - so the weaker answer this +/// function gives about the node beneath it is never the one acted on. Where the pass left the tree +/// alone, the proof held, and the answer asked here is the same one. A caller without these facts +/// (placement, which reads a tree without a binding) passes empty slices and gets the conservative +/// answer. #[allow(clippy::match_same_arms)] // Keep each expression's contribution guarantee explicit. -pub fn unifies_its_own_contributions(node: &NativeNode) -> Option { +pub fn unifies_its_own_contributions( + node: &NativeNode, + source_tags: &[Vec], + source_metrics: &[Option>], +) -> Option { match &node.expr { // A boundary hands over the pieces some upstream execution produced: what it holds is one // storage series at a time for a scan and one partition's fragments of a label set for an @@ -2510,7 +2951,20 @@ pub fn unifies_its_own_contributions(node: &NativeNode) -> Option { // arrive one whole storage series at a time, so it answers one series per series it read, // under the label set it read. `timestamp(x)` is the exception: it reports the sample's own // time under the metric name dropped, which can give two series one label set. - NativeExpr::Instant { op, .. } => Some(op.value != InstantValue::Timestamp), + NativeExpr::Instant { input, op } => { + if op.value == InstantValue::Timestamp { + // `timestamp` drops the metadata labels, so what it emits aliases unless the drop + // provably tells its series apart. The labels half is asked here; the keys half is + // what the pass collects with, above this node - see this function's docs. + Some(metadata_labels_keep_series_apart( + input, + source_tags, + source_metrics, + )) + } else { + Some(true) + } + } // The merge is the collection: whatever its input handed over, it groups the fragments by // canonical label set and emits one series per set, so what it emits shares none. NativeExpr::Merge { .. } => Some(true), @@ -2528,20 +2982,46 @@ pub fn unifies_its_own_contributions(node: &NativeNode) -> Option { // folds each contribution on its own, so what it emits does alias its input's fragments, // and `Some(false)` is exact. NativeExpr::Window { input, op } => { - Some(!op.function.drops_metric_name() && unifies_its_own_contributions(input).is_some()) + let child = unifies_its_own_contributions(input, source_tags, source_metrics); + if child.is_none() { + // A boundary hands the pieces of one series over one at a time, and the window folds + // them without ever seeing the whole series: what it emits aliases whatever the + // labels say. + Some(false) + } else if op.function.drops_metric_name() { + // The function drops the metadata labels, so the window answers `Some(true)` exactly + // when the labels say the drop tells its input's series apart. The keys half is the + // pass's to ask: where it fails, the pass has already put its merge above this + // window and the collector reads that merge's answer - see this function's docs. + Some(metadata_labels_keep_series_apart( + input, + source_tags, + source_metrics, + )) + } else { + Some(true) + } } - // A map can rewrite labels, so two series can end under one label set and the runtime - // records what it wrote in an `Emitted` table. + // A map can rewrite labels, so two series can end under one label set: what it hands out is + // not one series per set until a canonical merge has collected the set. NativeExpr::Map { .. } | NativeExpr::ParameterizedMap { .. } => Some(false), // `topk` hands back the series it read with their labels, so the input's answer is its own - // exactly as for the bound operation. - NativeExpr::ParameterizedTopK { input, .. } => unifies_its_own_contributions(input), + NativeExpr::ParameterizedTopK { input, .. } => { + unifies_its_own_contributions(input, source_tags, source_metrics) + } // A histogram function groups its rows by label set itself, so it emits one series per set. NativeExpr::ParameterizedHistogram { .. } => Some(true), // `scalar()` answers one value per point, which is not several series. NativeExpr::ScalarOf { .. } => Some(true), - // A timestamp reports a position under the metric name dropped. - NativeExpr::Timestamp { .. } => Some(false), + // A timestamp reports a position under the metadata labels dropped, and the labels half + // asked here is answered for the same reason the `timestamp` selection's is: the keys half + // is the one the pass turns into a merge above this node - see this function's docs. + NativeExpr::Timestamp { input } => Some(metadata_labels_keep_series_apart( + input, + source_tags, + source_metrics, + )), NativeExpr::Binary { left, right, @@ -2575,7 +3055,7 @@ pub fn unifies_its_own_contributions(node: &NativeNode) -> Option { !matches!(node.expr, NativeExpr::Scalar(BoundScalar { .. })) }; let vector = if carries_series(left) { left } else { right }; - unifies_its_own_contributions(vector) + unifies_its_own_contributions(vector, source_tags, source_metrics) } } // A set operation keeps each side's samples and files them by label set, one series per @@ -2603,7 +3083,7 @@ pub fn unifies_its_own_contributions(node: &NativeNode) -> Option { if whole { Some(true) } else { - unifies_its_own_contributions(input) + unifies_its_own_contributions(input, source_tags, source_metrics) } } } diff --git a/src/promql/src/native/plan/tests.rs b/src/promql/src/native/plan/tests.rs index 3ccea9949d7..5a321907c3f 100644 --- a/src/promql/src/native/plan/tests.rs +++ b/src/promql/src/native/plan/tests.rs @@ -24,7 +24,8 @@ use super::{ AggregateKind, AggregateStage, BoundAggregate, BoundInput, BoundInstant, BoundMap, BoundScalar, BoundVectorOp, BoundWindow, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, ParameterizedOp, aggregate_state_binding, duplicates_canonical_labels, - fused_output_tags, reads_across_series, unifies_its_own_contributions, + fused_output_tags, leaves_a_drop_to_the_storage_keys, reads_across_series, + unifies_its_own_contributions, }; use crate::extension_plan::FieldKind; use crate::extension_plan::{FusedSeries, FusedSeriesBinding}; @@ -187,6 +188,44 @@ fn window_over(inner: NativeNode, input_grid: Option>) -> NativeNo window_on(inner, input_grid, grid(0, 60_000, 4)) } +/// An instant selection over `inner` reporting the sample it read, which keeps its identity. +fn instant_sample(inner: NativeNode, out: Arc) -> NativeNode { + NativeNode { + expr: NativeExpr::Instant { + input: Box::new(inner), + op: BoundInstant { + lookback_ms: 300_000, + offset_ms: 0, + at_ms: None, + value: InstantValue::Sample, + output_grid: Arc::clone(&out), + }, + }, + output: evaluated(fields(1), out), + } +} + +/// A window over `inner` whose function keeps the metric name, so it hands its input's identity on. +fn keeping_window_fields(inner: NativeNode, out: Arc, count: usize) -> NativeNode { + NativeNode { + expr: NativeExpr::Window { + input: Box::new(inner), + op: BoundWindow { + function: RangeFunction::LastOverTime, + window: RangeWindow { + range_ms: 300_000, + offset_ms: 0, + anchor_ms: None, + modifier: None, + }, + input_grid: None, + output_grid: Arc::clone(&out), + }, + }, + output: evaluated(fields(count), out), + } +} + /// A window over `inner` that produces `count` fields on `out`, so a consumer of it can be given a /// layout the window itself holds legally. fn window_fields(inner: NativeNode, out: Arc, count: usize) -> NativeNode { @@ -1175,12 +1214,12 @@ fn a_broadcast_side_contributes_no_labels() { // broadcast reaches each series of its vector side as it is, so whether two series can end // under one label set is that side's answer, not the constant's. assert!( - duplicates_canonical_labels(&aliasing(), &source_tags), + duplicates_canonical_labels(&aliasing(), &source_tags, &[], true), "the fixture aliases: a rewrite of a label its input carries can fold two series into one" ); assert_eq!( - duplicates_canonical_labels(&aliasing(), &source_tags), - duplicates_canonical_labels(&clamp_max(aliasing()), &source_tags), + duplicates_canonical_labels(&aliasing(), &source_tags, &[], true), + duplicates_canonical_labels(&clamp_max(aliasing()), &source_tags, &[], true), "a scalar parameter carries no labels, so the map answers its input's identity" ); for node in [ @@ -1190,8 +1229,8 @@ fn a_broadcast_side_contributes_no_labels() { binary(eval_time(), clamp_max(aliasing())), ] { assert_eq!( - duplicates_canonical_labels(&aliasing(), &source_tags), - duplicates_canonical_labels(&node, &source_tags), + duplicates_canonical_labels(&aliasing(), &source_tags, &[], true), + duplicates_canonical_labels(&node, &source_tags, &[], true), "a broadcast keeps the series identity of its vector side" ); } @@ -1204,7 +1243,7 @@ fn a_broadcast_side_contributes_no_labels() { binary(vector_of_one(), aliasing()), ] { assert!( - !duplicates_canonical_labels(&node, &source_tags), + !duplicates_canonical_labels(&node, &source_tags, &[], true), "a vector-vector match answers one series per label set, whatever aliases below it" ); } @@ -2161,15 +2200,25 @@ fn scan_tags() -> Vec> { vec![vec!["host".to_string()]] } +/// The labels a scan bound to one metric reports: its tag, and the name its binding gives every +/// series it hands over, which is not one of its tag columns. +fn named_scan_tags() -> Vec> { + vec![vec!["host".to_string(), "__name__".to_string()]] +} + /// A rewrite that writes a label the input does not carry cannot make two series equal, which is /// what lets a query like `label_replace(x, "dst", "$1", "host", "(.*)")` keep every partition. #[test] fn a_fresh_destination_does_not_alias() { let out = grid(0, 60_000, 4); - let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1); + // A window that keeps the metric name hands its input's identity on, so the rewrite is the only + // node that could make two series equal. + let inner = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); assert!(!duplicates_canonical_labels( &rewritten(inner, "dst", "host"), - &scan_tags() + &scan_tags(), + &[], + true )); } @@ -2181,7 +2230,9 @@ fn a_rewrite_of_an_existing_label_aliases() { let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1); assert!(duplicates_canonical_labels( &rewritten(inner, "host", "host"), - &scan_tags() + &scan_tags(), + &[], + true )); } @@ -2195,20 +2246,26 @@ fn a_rewrite_of_the_metric_name_aliases_a_named_source() { let named = vec![vec!["host".to_string(), "__name__".to_string()]]; assert!(duplicates_canonical_labels( &rewritten(inner, "__name__", "__name__"), - &named + &named, + &[], + true )); // The delete-only form can fold the same two identities, so it is collected too. let inner = input_on(Arc::clone(&out)); assert!(duplicates_canonical_labels( &replaced(inner, "__name__", "", "__name__"), - &named + &named, + &[], + true )); } /// A name write is collected even above a kernel that dropped the name: the scan below `abs`, a /// `timestamp` or a range function can still hold several metrics whose ordinary labels are equal, -/// so writing one name puts those identities under one label set. A fresh ordinary destination is -/// still uncollected, because no scan can carry an ordinary label it does not declare. +/// so writing one name puts those identities under one label set. A fresh ordinary destination over +/// a kernel that keeps the name is still uncollected, because no scan can carry an ordinary label it +/// does not declare -- while over a name-dropping kernel the answer is `true` for the kernel's own +/// sake, which is what [`dropping_the_metric_name_is_a_new_source`] pins. #[test] fn a_name_rewrite_above_a_dropped_name_is_collected() { let out = grid(0, 60_000, 4); @@ -2231,16 +2288,21 @@ fn a_name_rewrite_above_a_dropped_name_is_collected() { ] { assert!( duplicates_canonical_labels( - &replaced(dropped.clone(), "__name__", "same", "missing"), - &named + &replaced(dropped, "__name__", "same", "missing"), + &named, + &[], + true ), "{name}: a name write may alias metrics the kernel left indistinguishable" ); - assert!( - !duplicates_canonical_labels(&rewritten(dropped, "dst", "host"), &named), - "{name}: a fresh ordinary destination cannot alias" - ); } + // The fresh destination is judged against a kernel that keeps the name, which is the only shape + // where the rewrite alone decides the answer. + let keeping = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); + assert!( + !duplicates_canonical_labels(&rewritten(keeping, "dst", "host"), &named, &[], true), + "a fresh ordinary destination cannot alias" + ); } /// The label walk keeps a name a rewrite writes and drops it wherever the kernel does, so the @@ -2412,20 +2474,189 @@ fn an_unknown_source_may_alias() { let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1); assert!(duplicates_canonical_labels( &rewritten(inner, "dst", "host"), - &[] + &[], + &[], + true )); } -/// Dropping the metric name is not aliasing within one metric: the remaining tags are the primary -/// key, so two series that agree on all of them are the same stored series. +/// Dropping the metric name is a source of duplicates in its own right, and a window that keeps it +/// is not. +/// +/// The name is part of the identity a merge groups by, so a kernel that computes a value of its own +/// and reports the name dropped can leave two series under one label set - but only when something +/// below it could be told apart by that name. +/// +/// A scan bound to one metric names every series it hands over, so no name of its tells two of them +/// apart, and the labels are the whole proof: the one metric is enough - as long as the scan's key +/// columns always decode a value. A drop removes every empty label value with the name, so a key +/// column that holds `NULL` in one series and `''` in another leaves both under one label set and +/// that half withholds the proof too. A scan that may hand over several metrics is collected, and so +/// is one carrying `__type__` or `__unit__` among its tags, because only those labels told the two +/// series apart. #[test] -fn dropping_the_metric_name_keeps_the_inputs_answer() { +fn dropping_the_metric_name_is_a_new_source() { let out = grid(0, 60_000, 4); - let plain = window_fields(input_with_fields(1), Arc::clone(&out), 1); - assert!(!duplicates_canonical_labels(&plain, &scan_tags())); + let single = [Some(Arc::::from("some_metric"))]; + let dropped = window_fields(input_with_fields(1), Arc::clone(&out), 1); + // A caller that says nothing about the metric is not proved. + assert!(duplicates_canonical_labels( + &dropped, + &named_scan_tags(), + &[], + true + )); + // The scan reads one metric, so the name it drops told none of its series apart. assert!(!duplicates_canonical_labels( - &map_over(rewritten(plain, "dst", "host")), - &scan_tags() + &dropped, + &named_scan_tags(), + &single, + true + )); + // A source that may hand over several metrics is not proved, and neither is one that carries a + // metadata label of its own: both can hold two series that only that label told apart. + assert!(duplicates_canonical_labels( + &dropped, + &named_scan_tags(), + &[None], + true + )); + for metadata in ["__type__", "__unit__"] { + let mut tags = named_scan_tags(); + tags[0].push(metadata.to_string()); + assert!( + duplicates_canonical_labels(&dropped, &tags, &single, true), + "{metadata}" + ); + } + // The same scan, with a key column that can hold `NULL` in one series and `''` in another: the + // empty value leaves with the name, so the one metric no longer proves the drop - and neither + // does it for `abs` or a `timestamp` above it, which report the name dropped as well. + assert!(duplicates_canonical_labels( + &dropped, + &named_scan_tags(), + &single, + false + )); + assert!(duplicates_canonical_labels( + &map_over(input_on(Arc::clone(&out))), + &named_scan_tags(), + &single, + false + )); + // A window that keeps the name passes its input's answer on instead. + let keeping = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); + assert!(!duplicates_canonical_labels( + &keeping, + &named_scan_tags(), + &[], + true + )); + // `timestamp` reports a sample's own time under the name dropped, in both of its forms. + let stamps = |inner: NativeNode| NativeNode { + expr: NativeExpr::Timestamp { + input: Box::new(inner), + }, + output: evaluated(fields(1), Arc::clone(&out)), + }; + assert!(duplicates_canonical_labels( + &stamps(input_on(Arc::clone(&out))), + &named_scan_tags(), + &[], + true + )); + assert!(duplicates_canonical_labels( + &stamps(input_on(Arc::clone(&out))), + &named_scan_tags(), + &[None], + true + )); + assert!(!duplicates_canonical_labels( + &stamps(input_on(Arc::clone(&out))), + &named_scan_tags(), + &single, + true + )); + // A sample-preserving selection keeps the name it read, so it is no source of its own. + assert!(!duplicates_canonical_labels( + &instant_sample(input_on(Arc::clone(&out)), Arc::clone(&out)), + &named_scan_tags(), + &[], + true + )); +} + +/// The plan-time counter's question: which statements drop the metadata labels over series whose +/// labels prove them apart and whose keys do not. +/// +/// The labels half is what the walk reads; the key half is stated by the caller, and a drop is +/// counted exactly when the labels prove it, nothing below it has collected already, and the scan's +/// key columns cannot tell the pair apart - a key column holding `NULL` in one series and `''` in +/// another, which is a fact about the physical table. This is the tree half of that count: the +/// planner pairs it with the fact it read from the resolved table. +#[test] +fn a_drop_the_labels_prove_is_counted_only_when_its_keys_do_not() { + let out = grid(0, 60_000, 4); + let single = [Some(Arc::::from("some_metric"))]; + let named = named_scan_tags(); + // `rate(x[5m])` and `abs(abs(x))` over one metric: the drop removes a name every series shares, + // and nothing below either node is collected, so the key columns are what decides the answer. + for keys in [false, true] { + let counted = !keys; + assert_eq!( + counted, + leaves_a_drop_to_the_storage_keys( + &NativeTree::try_new(window_fields(input_with_fields(1), Arc::clone(&out), 1)) + .unwrap(), + &named, + &single, + keys, + ), + "`rate(x[5m])` with keys_survive = {keys}" + ); + assert_eq!( + counted, + leaves_a_drop_to_the_storage_keys( + &NativeTree::try_new(map_over(map_over(input_on(Arc::clone(&out))))).unwrap(), + &named, + &single, + keys, + ), + "`abs(abs(x))` with keys_survive = {keys}" + ); + } + // A source that may hand over several metrics is collected by the labels, so nothing is left to + // the key columns. + assert!(!leaves_a_drop_to_the_storage_keys( + &NativeTree::try_new(window_fields(input_with_fields(1), Arc::clone(&out), 1)).unwrap(), + &named, + &[None], + false, + )); + // A window that keeps the name drops nothing. + assert!(!leaves_a_drop_to_the_storage_keys( + &NativeTree::try_new(keeping_window_fields( + input_with_fields(1), + Arc::clone(&out), + 1 + )) + .unwrap(), + &named, + &single, + false, + )); + // A merge below the drop hands over one series per canonical label set, which is unique even + // where the grouping treats an empty label value as no label, so the key columns decide nothing. + assert!(!leaves_a_drop_to_the_storage_keys( + &NativeTree::try_new(window_on( + merge_over(input_on(Arc::clone(&out))), + Some(Arc::clone(&out)), + Arc::clone(&out), + )) + .unwrap(), + &named, + &single, + false, )); } @@ -2434,7 +2665,8 @@ fn dropping_the_metric_name_keeps_the_inputs_answer() { #[test] fn a_rewrite_after_ranking_or_grouping_may_alias() { let out = grid(0, 60_000, 4); - let selected = || window_fields(input_with_fields(1), Arc::clone(&out), 1); + // A name-keeping window, so the ranking below is the only node whose answer is in question. + let selected = || keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); let vector = |op| NativeNode { expr: NativeExpr::Vector { input: Box::new(selected()), @@ -2475,7 +2707,10 @@ fn a_rewrite_after_ranking_or_grouping_may_alias() { ), ("sum by host", grouped_sum), ] { - assert!(!duplicates_canonical_labels(&inner, &source_tags), "{name}"); + assert!( + !duplicates_canonical_labels(&inner, &source_tags, &[], true), + "{name}" + ); let output = inner.output.clone(); let renamed = NativeNode { expr: NativeExpr::Map { @@ -2491,7 +2726,7 @@ fn a_rewrite_after_ranking_or_grouping_may_alias() { }; assert!(renamed.validate().is_ok(), "{name}"); assert!( - duplicates_canonical_labels(&renamed, &source_tags), + duplicates_canonical_labels(&renamed, &source_tags, &[], true), "{name}" ); } @@ -2611,7 +2846,9 @@ fn a_merge_answers_one_series_per_set() { ); assert!(!duplicates_canonical_labels( &merge_over(aliasing), - &scan_tags() + &scan_tags(), + &[], + true )); } @@ -2661,25 +2898,64 @@ fn a_window_unifies_its_own_contributions() { ] { assert_eq!( child_answer, - unifies_its_own_contributions(&child), + unifies_its_own_contributions(&child, &[], &[]), "{name}: the child's own answer" ); assert_eq!( window_answer, - unifies_its_own_contributions(&window_of(child, RangeFunction::LastOverTime)), + unifies_its_own_contributions(&window_of(child, RangeFunction::LastOverTime), &[], &[]), "{name}: a name-keeping window unifies whatever its child reported, and only a \ boundary leaves it aliasing" ); } - // A function that drops the metric name can give two series one label set whatever its child - // answered, so the window reports it whatever the child said. - for child in [collected(), fragments(), boundary()] { + // A function that drops the metadata labels can give two series one label set, so the window + // reports it whatever its child said - and a caller that cannot say which metric the child reads + // has no proof to offer on the child's behalf. + for child in [collected(), fragments()] { assert_eq!( Some(false), - unifies_its_own_contributions(&window_of(child, RangeFunction::Rate)), - "a name-dropping window may alias" + unifies_its_own_contributions(&window_of(child, RangeFunction::Rate), &[], &[]), + "a name-dropping window over an unproved source may alias" ); } + // A boundary is the one child the proof says nothing about: the window folds the pieces of one + // series one at a time, so what it emits aliases them whatever the labels say. + assert_eq!( + Some(false), + unifies_its_own_contributions(&window_of(boundary(), RangeFunction::Rate), &[], &[]), + "a boundary leaves the window's own answer aliasing" + ); + // The proof is the one the canonical pass reads: a child whose labels lost the name, over a + // source bound to a single metric without a metadata tag of its own, keeps the name-dropping + // window from reporting fragments. + let named = [vec!["host".to_string(), "__name__".to_string()]]; + let single = [Some(Arc::::from("some_metric"))]; + let dropped = map_over(input_on(Arc::clone(&out))); + assert_eq!( + Some(true), + unifies_its_own_contributions( + &window_of(dropped.clone(), RangeFunction::Rate), + &named, + &single + ), + "a single metric's dropped name tells none of its series apart" + ); + assert_eq!( + Some(false), + unifies_its_own_contributions( + &window_of(dropped.clone(), RangeFunction::Rate), + &named, + &[None] + ), + "a source that may hand over several metrics is not proved" + ); + let mut typed = named.clone(); + typed[0].push("__type__".to_string()); + assert_eq!( + Some(false), + unifies_its_own_contributions(&window_of(dropped, RangeFunction::Rate), &typed, &single), + "a metadata tag the window drops is not proved constant" + ); } /// `info` reads its info side through a canonical merge: the merge only collects the pieces of one diff --git a/src/promql/src/native/runs.rs b/src/promql/src/native/runs.rs deleted file mode 100644 index 5322221ffab..00000000000 --- a/src/promql/src/native/runs.rs +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The grid positions a field occupies, as sorted, disjoint runs. -//! -//! A node that rewrites labels — a `Map` dropping the metric name, a `Window` dropping the name and -//! the empty-valued labels — can turn two input series into one label set, and every layer above -//! then reads the two chunks as one series. What decides whether that pair is legal is which -//! positions of each field already hold a value: two values of one field at one position cannot be -//! merged away, while a chunk that fills positions the set left empty merges cleanly, as -//! [`crate::grid_series::merge_same_labelset`] does. -//! -//! The native runtime's partition-local [`crate::native::execute::Emitted`] table keeps this state -//! per label set and field as runs, so a chunk of many positions costs one run rather than one -//! entry per position. - -/// One run of grid positions, half-open, that hold a value in one field. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Run { - pub(crate) start: usize, - pub(crate) end: usize, -} - -impl Run { - /// Whether two runs share a position. - pub(crate) fn overlaps(self, other: Self) -> bool { - self.start < other.end && other.start < self.end - } -} - -// How many times `absorb` rebuilt the stored runs instead of appending to them. -// -// Test-only, and per thread: the runs stored afterwards are the same either way, so a rebuild shows -// only in what it costs, and a test that reads its own thread's figure is not disturbed by another -// test's chunks. -#[cfg(test)] -thread_local! { - static ABSORB_REBUILDS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// How many times this thread's [`absorb`] rebuilt the stored runs. -#[cfg(test)] -pub(crate) fn absorb_rebuilds() -> usize { - ABSORB_REBUILDS.with(std::cell::Cell::get) -} - -/// Merges `incoming` into `stored`, keeping the result in position order and joining the runs that -/// touch. -/// -/// Within a chunk one pass over both sides is what keeps this linear, since each side is sorted and -/// disjoint. Adding the runs one at a time and rebuilding the array after each would cost the square -/// of the runs a chunk yields: a chunk of one hundred thousand positions alternating between values -/// and nulls yields fifty thousand runs, and re-merging after each is over a billion element moves -/// for a single series. -/// -/// One pass per chunk is still a pass over everything the series has stored so far, so it is not -/// free once the *history* grows across chunks: a series split into `K` chunks of one position each, -/// a hole apart, stores one more run per chunk and moves `K(K+1)/2` of them, with no label collision -/// and no illegal input anywhere. A chunk that starts no earlier than the end of the last stored run -/// only ever adds to the end — the stored runs are sorted and disjoint, so every one of them ends -/// before anything this chunk holds — and it is appended instead, which is amortised constant per -/// run. That is the shape a scan hands over as its positions move forward chunk after chunk. A chunk -/// that starts before the last stored run can split a run or reach back past it, so it still costs -/// the full merge. -/// -/// Either way the result is sorted, disjoint, and has the runs that touch joined, which is what a -/// look-up by [`Run::overlaps`] on the runs stored here relies on. -pub(crate) fn absorb(stored: &mut Vec, incoming: &[Run]) { - if incoming.is_empty() { - return; - } - if stored - .last() - .is_none_or(|last| incoming[0].start >= last.end) - { - append_runs(stored, incoming); - } else { - merge_runs(stored, incoming); - } -} - -/// Appends `incoming` to `stored`, all of which lies past the last stored run. -/// -/// Nothing already stored has to be placed again: the two sides are each in position order and -/// nothing `incoming` holds can reach back into what is stored, so its runs are pushed on the end and -/// the vector's own growth is the only move — amortised over the runs appended. Where its first run -/// touches the end of the last stored run the two are one run, as they are after a merge, so it is -/// joined rather than left beside it. -fn append_runs(stored: &mut Vec, incoming: &[Run]) { - let mut incoming = incoming; - if let Some(last) = stored.last_mut() - && let Some(first) = incoming.first() - && first.start <= last.end - { - last.end = last.end.max(first.end); - incoming = &incoming[1..]; - } - stored.extend_from_slice(incoming); -} - -/// Merges `incoming` into `stored`, where the two can interleave. -/// -/// One pass is enough because each side is sorted and disjoint: whichever run starts first is -/// placed, and it joins the run before it where the two touch. -fn merge_runs(stored: &mut Vec, incoming: &[Run]) { - #[cfg(test)] - ABSORB_REBUILDS.with(|rebuilds| rebuilds.set(rebuilds.get() + 1)); - let mut merged: Vec = Vec::with_capacity(stored.len() + incoming.len()); - let (mut a, mut b) = (0, 0); - while a < stored.len() || b < incoming.len() { - let larger = match (stored.get(a), incoming.get(b)) { - (Some(left), Some(right)) if right.start < left.start => { - b += 1; - *right - } - (Some(left), _) => { - a += 1; - *left - } - (None, Some(right)) => { - b += 1; - *right - } - (None, None) => break, - }; - match merged.last_mut() { - Some(last) if larger.start <= last.end => last.end = last.end.max(larger.end), - _ => merged.push(larger), - } - } - *stored = merged; -} diff --git a/src/promql/src/native/segment.rs b/src/promql/src/native/segment.rs index d3dd151ad19..6a32984055d 100644 --- a/src/promql/src/native/segment.rs +++ b/src/promql/src/native/segment.rs @@ -268,21 +268,6 @@ fn cut_as_segment(node: &NativeNode, canonical_segmentation: bool) -> bool { !sources.is_empty() } -/// Whether any canonical merge in `node` answers with histogram samples, given the kind its input -/// carries. -/// -/// A merge collects the pieces of one label set and hands on the samples it was given, so its answer -/// kind is its child's. A histogram answer standing on a merge would travel through the canonical -/// boundary carrying histogram frames, a pairing no execution exercises yet: the planner keeps such -/// a statement on the existing plan until that pairing is covered, and it has to look at every -/// merge, not only a root one. -pub fn merge_answers_histogram(node: &NativeNode, input_kind: FieldKind) -> bool { - node.any(&|node| { - matches!(node.expr, NativeExpr::Merge { .. }) - && node.output_field_kind(Some(input_kind)) == Some(FieldKind::Histogram) - }) -} - /// Whether the merge producer's entire prefix can run on each scan partition independently. pub fn canonical_local_prefix(node: &NativeNode) -> bool { match &node.expr { diff --git a/src/promql/src/range_eval.rs b/src/promql/src/range_eval.rs index 00ef42ddf67..e0223b4ac58 100644 --- a/src/promql/src/range_eval.rs +++ b/src/promql/src/range_eval.rs @@ -200,12 +200,36 @@ pub struct RangeEvaluator { batch: Option, /// Windows accumulated for the next batch call. pending: PendingBatch, + /// Buffers the bulk ingest shifts and compacts a chunk into, kept between chunks so a series + /// that needs them does not allocate per push. + bulk_scratch: BulkScratch, /// Batch kernel calls made so far. Test-only: both dispatch shapes produce the same output, /// so a test comparing only output could not tell which one ran. #[cfg(test)] batch_calls: usize, } +/// The buffers the bulk ingest needs for a chunk whose samples are not retained as they arrive: +/// the timestamps shifted by the selector offset, and one compacted column per field. +#[derive(Default)] +struct BulkScratch { + timestamps: Vec, + columns: Vec>, +} + +impl BulkScratch { + /// The bytes these buffers hold, by capacity. + fn bytes(&self) -> usize { + self.timestamps.capacity() * std::mem::size_of::() + + self.columns.capacity() * std::mem::size_of::>() + + self + .columns + .iter() + .map(|column| column.capacity() * std::mem::size_of::()) + .sum::() + } +} + /// Windows accumulated for the next batch call: one list per field. struct PendingBatch { /// Grid index of each accumulated point, ascending. @@ -686,11 +710,20 @@ impl RangeEvaluator { bulk_ingests: 0, batch: None, pending: PendingBatch::with_fields(input.fields.len()), + bulk_scratch: BulkScratch::default(), #[cfg(test)] batch_calls: 0, }) } + /// The evaluation grid this evaluator decides points of. + /// + /// Every window ends at a point of it and is read at that point's timestamp, so a caller + /// reserving for what the evaluator is about to allocate reserves this grid's timestamps. + pub fn grid(&self) -> &Arc { + &self.grid + } + /// Chunks appended without the per-sample loop. Test-only: both paths produce the same /// output by construction, so a test that only compared output could not tell whether the /// bulk path ran at all. @@ -716,8 +749,9 @@ impl RangeEvaluator { /// Counted by capacity, so it is what the storage holds rather than what it uses: the retained /// samples per field (timestamps, values, the reset index and the native histograms with the /// payload each of them owns), the row a push fills with the histograms it boxes, the identity - /// of the open series and the pending batch's own vectors. Left out are the kernel's scratch - /// buffers, this struct's `Vec` headers and the grid, which every chunk addressed on it shares. + /// of the open series, the pending batch's own vectors and the buffers the bulk ingest shifts + /// and compacts into. Left out are the kernel's scratch buffers, this struct's `Vec` headers + /// and the grid, which every chunk addressed on it shares. /// /// The figure is an estimate, not a ledger of every allocation: a histogram's payload is /// measured once when it is admitted and released with it, so neither the retained samples nor @@ -735,6 +769,7 @@ impl RangeEvaluator { + boxed_bytes(&self.row) + self.series.as_ref().map_or(0, SeriesState::identity_bytes) + self.pending.bytes() + + self.bulk_scratch.bytes() } /// An upper bound of the rows the next push hands out, for input whose last sample is at @@ -1066,63 +1101,139 @@ impl RangeEvaluator { Ok(()) } - /// Appends the chunk in bulk when every field is an entirely present, unshifted float column. + /// Appends the chunk in bulk when every field is an entirely present float column. /// - /// Returns `false` when any condition of the per-sample path applies, so the caller can run - /// it. A chunk carrying a stale marker takes that path too: the marker drops the whole row, - /// and reproducing that here would cost the same scan the loop already makes. + /// Returns `false` when a condition of the per-sample path applies, so the caller can run it. + /// Only the shapes that path alone can read do: a field holding native histogram samples or + /// mixed floats and histograms -- `dense()` reports [`DenseSamplesRef::Mixed`] for the latter, + /// keeping its windows on the histogram path where the mixed-window rules decide the result -- + /// and a column with missing samples, which are absent rows rather than values. /// - /// A field mixing floats and native histograms never qualifies: `dense()` reports - /// [`DenseSamplesRef::Mixed`] for it, keeping its windows on the histogram path where the - /// mixed-window rules decide the result. + /// A selector offset and a stale marker do not: the shift and the rows the marker drops are + /// applied here, over the columns, instead of handing the chunk to the per-sample loop. The + /// search for a marker is one short-circuiting pass, the same pass the loop would have made. fn ingest_in_bulk( &mut self, timestamps: &[i64], fields: &SeriesFields, drop_stale: bool, - ) -> bool { + ) -> Result { // Lengths are checked by `check_chunk` for raw input and by `push_series` for evaluated // input, so only the shape of the columns is decided here. - if self.window.offset_ms != 0 { - return false; - } let mut columns = Vec::with_capacity(fields.len()); for field in fields.iter() { let Some(DenseSamplesRef::Float(values)) = field.dense() else { - return false; + return Ok(false); }; if values.null_count() != 0 { - return false; + return Ok(false); } - columns.push(values); + columns.push(values.values().as_ref()); } - // A kept sample's shifted time must fit `i64`; with no offset it is the source timestamp, - // which `RawTimestamps` already validated as increasing and which the frame widened into - // this array without narrowing, so there is nothing left to check here. - if drop_stale - && columns.iter().any(|values| { - values - .values() - .iter() - .any(|value| is_prometheus_stale_nan(*value)) - }) + // A stale marker drops its whole row, so which rows go is decided before anything is + // retained. A chunk holding none -- the common case -- is judged by one scan that stops at + // the first marker and then leaves the columns as they are. + let stale_rows = if drop_stale + && columns + .iter() + .any(|values| values.iter().any(|value| is_prometheus_stale_nan(*value))) { - return false; - } - for (field, values) in self.fields.iter_mut().zip(columns) { - field.push_bulk(timestamps, values.values()); - } + let mut stale = vec![false; timestamps.len()]; + for values in &columns { + for (row, dropped) in stale.iter_mut().enumerate() { + *dropped |= is_prometheus_stale_nan(values[row]); + } + } + Some(stale) + } else { + None + }; + self.append_float_chunk(timestamps, &columns, stale_rows.as_deref())?; #[cfg(test)] { self.bulk_ingests += 1; } - true + Ok(true) + } + + /// Retains one chunk of entirely present float columns. + /// + /// `stale` marks the rows a stale marker drops, `None` when the chunk holds none. The retained + /// samples are the chunk's rows less those, on the evaluation timeline: the selector offset + /// shifts them, and only a chunk that needs neither keeps the incoming timestamps as they are. + fn append_float_chunk( + &mut self, + timestamps: &[i64], + columns: &[&[f64]], + stale: Option<&[bool]>, + ) -> Result<()> { + let offset_ms = self.window.offset_ms; + if offset_ms == 0 && stale.is_none() { + // The source timestamps, which `RawTimestamps` already validated as increasing and + // which the frame widened into this array without narrowing: nothing to shift, nothing + // to check, and the two appends become two `memcpy`s. + for (field, values) in self.fields.iter_mut().zip(columns) { + field.push_bulk(timestamps, values); + } + return Ok(()); + } + let BulkScratch { + timestamps: mut shifted, + columns: mut compacted, + } = std::mem::take(&mut self.bulk_scratch); + shifted.clear(); + shifted.reserve(timestamps.len()); + for (row, ×tamp) in timestamps.iter().enumerate() { + if stale.is_some_and(|stale| stale[row]) { + continue; + } + // Like RangeManipulate, any kept sample whose shifted time leaves i64 fails the series, + // whichever path retains it. + let Some(shifted_timestamp) = timestamp.checked_add(offset_ms) else { + self.bulk_scratch = BulkScratch { + timestamps: shifted, + columns: compacted, + }; + return Err(DataFusionError::Execution(format!( + "sample timestamp {timestamp} overflows with offset {offset_ms} ms" + ))); + }; + shifted.push(shifted_timestamp); + } + match stale { + None => { + for (field, values) in self.fields.iter_mut().zip(columns) { + field.push_bulk(&shifted, values); + } + } + Some(stale) => { + compacted.resize_with(columns.len(), Vec::new); + for (retained, values) in compacted.iter_mut().zip(columns) { + retained.clear(); + retained.extend( + values + .iter() + .enumerate() + .filter(|(row, _)| !stale[*row]) + .map(|(_, &value)| value), + ); + } + for (field, values) in self.fields.iter_mut().zip(&compacted) { + field.push_bulk(&shifted, values); + } + } + } + self.bulk_scratch = BulkScratch { + timestamps: shifted, + columns: compacted, + }; + Ok(()) } /// Retains the samples of `fields` at `timestamps`. From `raw` input a row holding a stale /// marker in any field is dropped; evaluated input keeps it as a value. fn ingest(&mut self, timestamps: &[i64], fields: &SeriesFields, raw: bool) -> Result<()> { - if self.ingest_in_bulk(timestamps, fields, raw) { + if self.ingest_in_bulk(timestamps, fields, raw)? { return Ok(()); } let mut samples: Vec<_> = fields.iter().map(|field| field.iter()).collect(); diff --git a/src/promql/src/range_eval/function.rs b/src/promql/src/range_eval/function.rs index ce54432ed28..db8ea8d3782 100644 --- a/src/promql/src/range_eval/function.rs +++ b/src/promql/src/range_eval/function.rs @@ -18,10 +18,28 @@ //! operation order. Arithmetic that GreptimeDB's range UDFs compute the same way is shared with //! them. +use std::collections::VecDeque; +use std::ops::Range; + use crate::functions::{EdgeKind, count_edges, extremum, instant_delta, kahan_inc}; use crate::go_sort; use crate::range_eval::{BatchKernel, FloatWindow, FloatWindows, RangeModifier, WindowKernel}; +/// The samples the batch kernels examine, counted per value. +/// +/// A batch kernel exists so a field is walked once per batch instead of once per window, which the +/// results cannot show: both shapes compute the same values. Counting what a batch kernel touches +/// is what tells them apart. +#[cfg(test)] +pub(super) static SAMPLES_EXAMINED: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +/// Counts `count` samples examined by a batch kernel; a no-op outside tests. +fn examined(_count: usize) { + #[cfg(test)] + SAMPLES_EXAMINED.fetch_add(_count as u64, std::sync::atomic::Ordering::Relaxed); +} + /// A PromQL range-vector function over float and native histogram samples. #[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] pub enum RangeFunction { @@ -91,28 +109,34 @@ impl RangeFunction { /// Whether the evaluator has to index counter resets for this function. /// - /// This must hold for exactly the kernels that read [`FloatWindow::counter_resets`], which is - /// the same as saying `IS_COUNTER` in `extrapolated_rate`: `rate` and `increase`. A kernel - /// that corrects counters without the index being maintained would silently drop every - /// correction, so a new counter function has to be added here as well. + /// This must hold for exactly the functions that read the index: `rate` and `increase`, whose + /// kernels read [`FloatWindow::counter_resets`] -- the same as saying `IS_COUNTER` in + /// `extrapolated_rate` -- and `resets`, whose batch kernel reads [`FloatWindows::resets_of`]. + /// A kernel that corrects or counts counters without the index being maintained would silently + /// miss every reset, so a new reset-reading function has to be added here as well. /// - /// It governs both readers of the index: the per-window kernel above, and the batch kernel, - /// which reads [`FloatWindows::resets_of`]. When it returns `false`, `resets` stays empty and - /// `resets_of` yields nothing, so a batch kernel for a counter function would skip every - /// correction and report `delta`-like values. + /// It governs both readers of the index. When it returns `false`, `resets` stays empty, + /// `resets_of` yields nothing and `counter_resets` is empty, so a kernel reading either would + /// see a series that never reset. pub(crate) fn indexes_counter_resets(self) -> bool { - matches!(self, Self::Rate | Self::Increase) + matches!(self, Self::Rate | Self::Increase | Self::Resets) } /// The batch form of this function, when it has one. /// - /// Only the counter functions have one. The rest keep the per-window form, which the - /// evaluator falls back to, so adding a function here is what changes which shape its - /// evaluation takes. + /// A batch form exists for the functions whose window answer is a running summary of the field + /// -- the two endpoints and the resets for the counter rates, a movement of the window's bounds + /// for the extremes, the resets inside it, and the change edges inside it. The rest keep the + /// per-window form, which the evaluator falls back to, so adding a function here is what + /// changes which shape its evaluation takes. pub(crate) fn batch_kernel(self) -> Option { match self { Self::Rate => Some(Box::new(extrapolated_rate_batch::)), Self::Increase => Some(Box::new(extrapolated_rate_batch::)), + Self::MinOverTime => Some(extremum_batch::()), + Self::MaxOverTime => Some(extremum_batch::()), + Self::Resets => Some(Box::new(resets_batch)), + Self::Changes => Some(changes_batch()), _ => None, } } @@ -488,6 +512,189 @@ pub(super) struct ExtrapolationSpan { pub samples_minus_one: usize, } +/// The batch form of `min_over_time` (`IS_MIN`) and `max_over_time`: one pass over the union of +/// the batch's windows, carrying the candidates that can still become the window's extremum. +/// +/// The windows of a batch start and end no earlier than the ones before them, so a sample that +/// lost to a later, strictly better one can never win a later window either: it leaves the back of +/// the queue, and a sample the window start has passed leaves the front. That is exactly what +/// [`extremum`] computes window by window, which is why the deque keeps the arrival order of equal +/// values (and with it the sign of tied zeros) and why a NaN never wins while the window holds +/// anything else. +fn extremum_batch() -> BatchKernel { + let mut candidates = SlidingExtremum::::default(); + Box::new( + move |batch: &FloatWindows<'_>, out: &mut Vec>| { + out.clear(); + out.reserve(batch.windows.len()); + // Batches are independent: the retained arrays of the previous one may have been drained + // since, and the samples behind the first window are gone anyway. + candidates.clear(); + let values = batch.values; + let sliding = pays_to_slide(batch.windows); + for span in batch.windows { + let (start, end) = (span.start, span.end); + out.push(if sliding { + candidates.evaluate(values, start, end) + } else if start == end { + None + } else { + examined(end - start); + extremum(values[start..end].iter().copied(), IS_MIN) + }); + } + }, + ) +} + +/// The samples between the first window's start and the last window's end: what a window-moving +/// summary of a batch has to walk, against the sum of the windows' lengths a rescan walks. +fn union_len(windows: &[Range]) -> usize { + match (windows.first(), windows.last()) { + (Some(first), Some(last)) => last.end.saturating_sub(first.start), + _ => 0, + } +} + +/// Whether walking the union of a batch's windows once beats scanning each window on its own. +/// +/// Reuse pays off in proportion to how much consecutive windows overlap: a window-moving summary +/// visits every sample between the first window's start and the last one's end -- plus a step per +/// window -- while a scan visits every sample of every window. On wide steps that barely overlap +/// the walk is the more expensive of the two. +fn pays_to_slide(windows: &[Range]) -> bool { + let scanned: usize = windows.iter().map(|span| span.len()).sum(); + union_len(windows) + windows.len() < scanned +} + +/// The extrema candidates of one window, carried to the next. +/// +/// Consecutive windows ascend, so the queue is only trimmed at the front and appended at the back. +#[derive(Default)] +struct SlidingExtremum { + /// Indices of the samples that can still become the extremum, in arrival order. A strictly + /// better sample evicts the ones queued before it, so the front is the window's extremum. + candidates: VecDeque, + /// Backs the all-NaN window, which reports the last NaN it holds, as [`extremum`] does. + latest_nan: Option, + /// The window the queue describes; `None` until the first window of a batch. + previous: Option<(usize, usize)>, +} + +impl SlidingExtremum { + fn clear(&mut self) { + self.candidates.clear(); + self.latest_nan = None; + self.previous = None; + } + + /// Whether `value` replaces `current`, the comparison [`extremum`] replaces on. + #[inline] + fn beats(value: f64, current: f64) -> bool { + if IS_MIN { + value < current + } else { + value > current + } + } + + /// The extremum of `values[start..end]`, `None` when the window holds no sample. + fn evaluate(&mut self, values: &[f64], start: usize, end: usize) -> Option { + let append_start = match self.previous { + Some((previous_start, previous_end)) + if start >= previous_start && end >= previous_end => + { + while self.candidates.front().is_some_and(|&index| index < start) { + self.candidates.pop_front(); + } + if self.latest_nan.is_some_and(|index| index < start) { + self.latest_nan = None; + } + previous_end.max(start) + } + // A retreating window would need samples the queue no longer holds. A batch's windows + // ascend, so this is the state a call starts from, not a case the walk reaches. + Some(_) | None => { + self.clear(); + start + } + }; + for (offset, &value) in values[append_start..end].iter().enumerate() { + if value.is_nan() { + self.latest_nan = Some(append_start + offset); + continue; + } + while let Some(&back) = self.candidates.back() { + if !Self::beats(value, values[back]) { + break; + } + self.candidates.pop_back(); + } + self.candidates.push_back(append_start + offset); + } + examined(end.saturating_sub(append_start)); + self.previous = Some((start, end)); + self.candidates + .front() + .or(self.latest_nan.as_ref()) + .map(|&index| values[index]) + } +} + +/// The batch form of `changes`: a prefix count of the change edges between neighbouring samples, +/// so a window's answer is the difference of two entries instead of a scan. +/// +/// The edges are counted with [`count_edges`], the predicate the per-window kernel uses, so the +/// two shapes agree on what a change is -- including that two NaNs, whose comparison is always +/// false, are not one. +fn changes_batch() -> BatchKernel { + let mut prefix: Vec = Vec::new(); + Box::new( + move |batch: &FloatWindows<'_>, out: &mut Vec>| { + out.clear(); + out.reserve(batch.windows.len()); + let Some((first, last)) = batch.windows.first().zip(batch.windows.last()) else { + return; + }; + let (union_start, union_end) = (first.start, last.end); + let scan = !pays_to_slide(batch.windows); + // `prefix[i]` counts the change edges among `values[union_start..=i]`, so the window + // `[start, end)` counts the edges inside it: its first one is the edge into `start + 1`. + prefix.clear(); + if !scan { + prefix.push(0); + examined(union_end.saturating_sub(union_start)); + for index in union_start + 1..union_end { + let edge = count_edges(&batch.values[index - 1..=index], EdgeKind::Changes); + prefix.push(prefix[prefix.len() - 1] + edge); + } + } + for span in batch.windows { + out.push((span.start != span.end).then(|| { + if scan { + examined(span.len()); + return count_edges(&batch.values[span.start..span.end], EdgeKind::Changes) + as f64; + } + let below = prefix[span.start - union_start]; + let inside = prefix[span.end - 1 - union_start]; + (inside - below) as f64 + })); + } + }, + ) +} + +/// The batch form of `resets`: the resets inside each window, read off the evaluator's reset +/// index. No sample is scanned: a window's answer is a count of index entries between its bounds. +fn resets_batch(batch: &FloatWindows<'_>, out: &mut Vec>) { + out.clear(); + out.reserve(batch.windows.len()); + for (index, span) in batch.windows.iter().enumerate() { + out.push((span.start != span.end).then(|| batch.resets_of(index).len() as f64)); + } +} + /// The factor `extrapolatedRate` scales the sampled increase by. `zero_point` is the first value /// over the increase, given only when Prometheus extrapolates a counter to zero: it is the share of /// the sampled interval that lies before `span.sampled_from`, so `0` says the counter's zero point diff --git a/src/promql/src/range_eval/histogram.rs b/src/promql/src/range_eval/histogram.rs index 3842226db82..196d8e2bb33 100644 --- a/src/promql/src/range_eval/histogram.rs +++ b/src/promql/src/range_eval/histogram.rs @@ -159,7 +159,8 @@ pub(super) fn evaluate( HistogramAggregate::Avg }; // The hints are read from the window's own histograms, so nothing is cloned for the - // check; the fold below clones one sample at a time as it consumes them. + // check; the fold below reads them by reference too and copies only the accumulator + // it starts from. if has_contradictory_reset_hints(histograms.iter().map(|(_, h)| h)) { record_warning( collector, @@ -173,7 +174,7 @@ pub(super) fn evaluate( // flag is the reconciliation its own adds already observed. Re-walking the pairs to // learn either would double the histogram work of every window. let (result, custom_reconciled) = - sum_or_avg(op, histograms.iter().map(|(_, h)| Ok(h.clone())))?; + sum_or_avg(op, histograms.iter().map(|(_, h)| Ok(Cow::Borrowed(h))))?; if result.is_none() { record_warning( collector, diff --git a/src/promql/src/range_eval/tests.rs b/src/promql/src/range_eval/tests.rs index de6d3fcf269..423fea39951 100644 --- a/src/promql/src/range_eval/tests.rs +++ b/src/promql/src/range_eval/tests.rs @@ -38,6 +38,7 @@ use datatypes::series::{Label, SeriesLabels}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; +use super::function::SAMPLES_EXAMINED; use super::{ CounterReset, FieldSamples, FloatWindow, POSITION_COMPARISONS, PendingBatch, RangeEvaluator, RangeFunction, RangeModifier, RangeWindow, WindowKernel, accumulate, subquery_grid, @@ -1923,6 +1924,237 @@ fn a_function_without_a_batch_form_stays_on_the_per_window_path() { assert_eq!(evaluator.batch_call_count(), 0); } +/// Rows of a counter that resets twice, plateaus once, steps unevenly and holds one stale marker: +/// every shape an extremum, a reset count or a change count can get wrong at a window bound. +fn summary_rows(points: usize) -> Vec { + let mut value = 0.0; + (0..points) + .map(|index| { + value = match index { + 200 | 500 => 1.0, + 300 => value, + _ if index % 3 == 0 => value + 2.0, + _ => value + 1.0, + }; + let sample = if index == 150 { stale() } else { value }; + (index as i64 * 10_000, vec![Some(sample)]) + }) + .collect() +} + +/// The functions whose window answer is a running summary of the field -- the extremes, the resets +/// and the changes -- reach the batch kernel, and its answers equal the per-window ones. +/// +/// The series is longer than `EVALUATION_BATCH`, so the batch also flushes in the middle of the +/// evaluation and has to carry the summary across the flush. A summary read off the wrong window +/// bounds, or a reset counted across one, differs from the per-window result on these rows. +#[test] +fn window_summaries_reach_the_batch_kernel() { + let points = 700; + let rows = summary_rows(points); + let window = RangeWindow { + range_ms: 40_000, + offset_ms: 0, + anchor_ms: None, + modifier: None, + }; + for function in [ + RangeFunction::MinOverTime, + RangeFunction::MaxOverTime, + RangeFunction::Resets, + RangeFunction::Changes, + ] { + let mut batch = RangeEvaluator::try_new( + &raw_schema(1), + grid(0, 10_000, points as u64), + window, + function, + ) + .unwrap(); + // The per-window path is the same evaluator without the batch kernel, which leaves the + // rest of the evaluation identical. + let mut per_window = RangeEvaluator::try_new( + &raw_schema(1), + grid(0, 10_000, points as u64), + window, + function, + ) + .unwrap(); + per_window.batch = None; + + let piece = chunk(&rows, 1, 0, true); + let batched = collect_chunks(&mut batch, std::slice::from_ref(&piece)); + let expected = collect_chunks(&mut per_window, std::slice::from_ref(&piece)); + assert!( + batch.batch_call_count() > 0, + "{function:?} never reached its batch kernel" + ); + assert_same_series( + fold_chunks(&batched, batch.output_schema()).as_ref(), + fold_chunks(&expected, per_window.output_schema()).as_ref(), + &format!("{function:?}"), + ); + } +} + +/// A batch kernel walks a field once per batch, not once per grid point. +/// +/// The per-window form visits every sample of every window, so `points` windows of `width` samples +/// visit `points * width`. A batch visits the union of its windows once plus one entry per window, +/// so the total stays close to the samples the series holds whatever the width is. The figure is +/// counted rather than timed, which is what makes it a property of the code and not of the +/// machine. +#[test] +fn window_summaries_are_not_rescanned_per_step() { + let points = 2_048u64; + let width = 60; + let rows = summary_rows(points as usize); + let window = RangeWindow { + range_ms: width as i64 * 10_000, + offset_ms: 0, + anchor_ms: None, + modifier: None, + }; + // What rescanning each window would visit, and what a walk of the retained samples costs. + let rescanned = points * width; + let once_over = points * 2; + let examined = |function| -> u64 { + SAMPLES_EXAMINED.store(0, std::sync::atomic::Ordering::Relaxed); + let mut evaluator = + RangeEvaluator::try_new(&raw_schema(1), grid(0, 10_000, points), window, function) + .unwrap(); + evaluator.push(&chunk(&rows, 1, 0, true)).unwrap(); + SAMPLES_EXAMINED.load(std::sync::atomic::Ordering::Relaxed) + }; + for function in [ + RangeFunction::MinOverTime, + RangeFunction::MaxOverTime, + RangeFunction::Changes, + ] { + let visits = examined(function); + assert!( + visits > 0, + "{function:?} reached no batch kernel, so nothing was counted" + ); + assert!( + visits <= once_over, + "{function:?} examined {visits} samples over {points} grid points, more than the \ + {once_over} a single walk of the series costs: the windows are being rescanned" + ); + } + // `resets` reads the reset index, so it touches no sample at all. + assert_eq!( + examined(RangeFunction::Resets), + 0, + "resets scanned samples instead of reading the reset index" + ); + assert!(rescanned > once_over * 10, "the comparison proves nothing"); +} + +/// The samples of a chunk kept in segments, which `dense()` refuses: the per-sample loop is the +/// only path that retains them, so one chunk can be handed to either path unchanged. +fn segmented(piece: &RawSeriesChunk, split: usize) -> RawSeriesChunk { + let mut piece = piece.clone(); + let len = piece.time.len(); + let fields = piece + .fields + .iter() + .map(|field| { + Samples::try_from_chunks(vec![ + field.slice(0, split).unwrap(), + field.slice(split, len - split).unwrap(), + ]) + .unwrap() + }) + .collect(); + piece.fields = SeriesFields::try_from_vec(fields).unwrap(); + piece +} + +/// A chunk read with a selector offset is appended in bulk, with the shift applied there rather +/// than by handing the whole chunk to the per-sample loop. +/// +/// The same rows go to two evaluators, one from a single dense chunk -- which the bulk path takes +/// -- and one from a chunk kept in segments, which only the per-sample loop retains. The results +/// have to be identical, and only the first may count a bulk ingest. +#[test] +fn an_offset_chunk_is_appended_in_bulk() { + let rows: Vec = (0..5) + .map(|index| (index as i64 * 10_000, vec![Some(index as f64)])) + .collect(); + let window = RangeWindow { + range_ms: 20_000, + offset_ms: 10_000, + anchor_ms: None, + modifier: None, + }; + let evaluate = |piece: &RawSeriesChunk| { + let mut evaluator = RangeEvaluator::try_new( + &raw_schema(1), + grid(0, 10_000, 5), + window, + RangeFunction::SumOverTime, + ) + .unwrap(); + let chunks = collect_chunks(&mut evaluator, std::slice::from_ref(piece)); + ( + fold_chunks(&chunks, evaluator.output_schema()), + evaluator.bulk_ingest_count(), + ) + }; + let piece = chunk(&rows, 1, 0, true); + let (bulk, bulk_ingests) = evaluate(&piece); + let (per_sample, per_sample_ingests) = evaluate(&segmented(&piece, 2)); + assert_eq!( + bulk_ingests, 1, + "a chunk shifted by an offset was handed to the per-sample loop" + ); + assert_eq!( + per_sample_ingests, 0, + "the segmented chunk took the bulk path, so the two were not compared" + ); + assert_same_series(bulk.as_ref(), per_sample.as_ref(), "offset"); +} + +/// A chunk holding a stale marker is appended in bulk too, with the rows the marker drops taken +/// out there, and the result is what retaining the same rows sample by sample produces. +#[test] +fn a_stale_marker_does_not_force_the_per_sample_loop() { + let mut rows: Vec = (0..5) + .map(|index| (index as i64 * 10_000, vec![Some(index as f64)])) + .collect(); + rows[2].1[0] = Some(stale()); + let window = RangeWindow { + range_ms: 20_000, + offset_ms: 0, + anchor_ms: None, + modifier: None, + }; + let evaluate = |piece: &RawSeriesChunk| { + let mut evaluator = RangeEvaluator::try_new( + &raw_schema(1), + grid(0, 10_000, 5), + window, + RangeFunction::LastOverTime, + ) + .unwrap(); + let chunks = collect_chunks(&mut evaluator, std::slice::from_ref(piece)); + ( + fold_chunks(&chunks, evaluator.output_schema()), + evaluator.bulk_ingest_count(), + ) + }; + let piece = chunk(&rows, 1, 0, true); + let (bulk, bulk_ingests) = evaluate(&piece); + let (per_sample, per_sample_ingests) = evaluate(&segmented(&piece, 2)); + assert_eq!( + bulk_ingests, 1, + "a chunk holding a stale marker was handed to the per-sample loop" + ); + assert_eq!(per_sample_ingests, 0); + assert_same_series(bulk.as_ref(), per_sample.as_ref(), "stale marker"); +} + #[test] fn empty_input_and_series_without_results() { let window = RangeWindow { diff --git a/src/promql/src/select.rs b/src/promql/src/select.rs index 1ffcc1cf9f4..50c66a33e16 100644 --- a/src/promql/src/select.rs +++ b/src/promql/src/select.rs @@ -32,7 +32,7 @@ use datatypes::series::time::{EvalGrid, GridRange, TimeAxis, union_positions}; use datatypes::series::{Label, SeriesLabels}; use crate::go_sort; -use crate::grid_series::{GridSeriesBuilder, external, owned}; +use crate::grid_series::{GridSeriesBuilder, external}; #[cfg(test)] mod tests; @@ -90,17 +90,28 @@ pub fn topk_at( // positions, not the whole grid. grid_points(schema, series)?; let mut order: Vec = (0..series.len()).collect(); - order.sort_by(|&a, &b| compare_labels(&series[a].metric, &series[b].metric)); + // The label pairs of a series are built once: a comparator that compared two metrics would + // rebuild both lists per comparison, and the groups read the same lists again. + let mut label_sets: Vec, Arc)>> = series + .iter() + .map(|item| labels_with_name(&item.metric)) + .collect(); + order.sort_by(|&a, &b| label_sets[a].cmp(&label_sets[b])); let mut groups: HashMap> = HashMap::new(); for index in order { groups - .entry(group_key(&series[index].metric, grouping)) + .entry(group_key_from(&label_sets[index], grouping)) .or_default() .push(index); } + // The groups hold what they need, so the label lists are freed before the point walk. + label_sets.clear(); let mut selected: Vec> = vec![Vec::new(); series.len()]; let mut candidates: Vec<(usize, usize, f64)> = Vec::new(); + // The values of one point, for the partial selection below: ranking them there leaves the + // candidates in member order, which is what hands ties to the earlier label set. + let mut values: Vec = Vec::new(); for members in groups.values() { // A point no member carries holds no candidate, so only the union of the members' // positions is visited: a hole in the grid never reaches the ranking. Each member reads @@ -122,12 +133,41 @@ pub fn topk_at( candidates.push((member, row, value)); } } - // Members are in label order, so a stable sort hands ties to the earlier label set. - candidates.sort_by(|a, b| rank(a.2, b.2, top)); let Some(k) = k_at(point)? else { continue; }; - for &(member, row, _) in candidates.iter().take(k) { + // Which candidates win is the result; the order they were found in is not. The ranking + // is a partial selection of the k best values rather than an ordering of the whole + // group, and it runs on a copy of the values so that the candidates keep the member + // order the ties are resolved in. + let k = k.min(candidates.len()); + if k == 0 { + continue; + } + let boundary = (k < candidates.len()).then(|| { + values.clear(); + values.extend(candidates.iter().map(|candidate| candidate.2)); + values.select_nth_unstable_by(k, |a, b| rank(*a, *b, top)); + values[k] + }); + // The partition splits the run of values equal to its boundary, while ties go to the + // earlier label set: the ones equal to the boundary inside the front are the slots left + // for them, and the scan below fills those slots in member order, which is label order. + let mut tied = match boundary { + Some(boundary) => values[..k] + .iter() + .filter(|&&value| rank(value, boundary, top) == Ordering::Equal) + .count(), + None => 0, + }; + for &(member, row, value) in &candidates { + if let Some(boundary) = boundary { + match rank(value, boundary, top) { + Ordering::Less => {} + Ordering::Equal if tied > 0 => tied -= 1, + _ => continue, + } + } selected[member].push((point, row)); } } @@ -218,8 +258,9 @@ pub fn limitk( // A series belongs to one group, whose points are visited in order. let mut cursor = series[index].fields.as_slice()[0].cursor(); for (point, row) in points { + // The sample is moved, not computed, so it goes straight into the result. let sample = cursor.at(row).expect("the axis and the fields are aligned"); - builder.push_samples(point, &[owned(sample)?])?; + builder.push_refs(point, &[sample])?; } result.extend(builder.finish(series[index].metric.clone())?); } @@ -240,7 +281,7 @@ pub fn limit_ratio(ratio: f64, schema: &SeriesSchema, series: &[Series]) -> Resu let ratio = ratio.clamp(-1.0, 1.0); let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone()))?; let mut result = Vec::new(); - let mut row = Vec::with_capacity(schema.fields.len()); + let mut row: Vec> = Vec::with_capacity(schema.fields.len()); for item in series { let offset = sample_offset(&item.metric); let kept = if ratio >= 0.0 { @@ -257,14 +298,15 @@ pub fn limit_ratio(ratio: f64, schema: &SeriesSchema, series: &[Series]) -> Resu item.fields.iter().map(|field| field.cursor()).collect(); for (point, index) in item.time.positions() { row.clear(); + // The samples are moved, not computed, so the row is read and pushed as it stands. for cursor in &mut cursors { - row.push(owned( + row.push( cursor .at(index) .expect("the axis and the fields are aligned"), - )?); + ); } - builder.push_samples(point, &row)?; + builder.push_refs(point, &row)?; } result.extend(builder.finish(item.metric.clone())?); } @@ -538,6 +580,8 @@ const GO_NAN: f64 = f64::from_bits(0x7FF8_0000_0000_0001); /// The order of `a` and `b` for `topk` (`top`) or `bottomk`; NaN ranks last for both. fn rank(a: f64, b: f64, top: bool) -> Ordering { + #[cfg(test)] + calls::record_rank(); match (a.is_nan(), b.is_nan()) { (true, true) => Ordering::Equal, (true, false) => Ordering::Greater, @@ -608,8 +652,13 @@ pub(crate) fn validate_topk_parameters(values: &[Option]) -> Result<()> { type GroupKey = Vec<(Arc, Arc)>; fn group_key(metric: &SeriesLabels, grouping: &Grouping) -> GroupKey { - let mut key: Vec<_> = labels_with_name(metric) - .into_iter() + group_key_from(&labels_with_name(metric), grouping) +} + +/// The group `labels` belong to, for callers that already hold the series' label pairs. +fn group_key_from(labels: &[(Arc, Arc)], grouping: &Grouping) -> GroupKey { + let mut key: Vec<_> = labels + .iter() .filter(|(name, value)| { !value.is_empty() && match grouping { @@ -617,6 +666,7 @@ fn group_key(metric: &SeriesLabels, grouping: &Grouping) -> GroupKey { Grouping::Without(names) => &**name != "__name__" && !names.contains(name), } }) + .cloned() .collect(); key.sort(); key @@ -624,6 +674,8 @@ fn group_key(metric: &SeriesLabels, grouping: &Grouping) -> GroupKey { /// The label pairs of `metric` with `__name__` among them, sorted by name. fn labels_with_name(metric: &SeriesLabels) -> Vec<(Arc, Arc)> { + #[cfg(test)] + calls::record_label_list(); let mut labels: Vec<_> = metric .tags() .iter() @@ -754,3 +806,38 @@ fn into_value(sample: SampleRef<'_>) -> Result> { fn cursor_value(cursor: &mut RowCursor<'_>, row: usize) -> Result> { into_value(cursor.at(row).expect("the axis and the fields are aligned")) } + +/// How often this thread built a series' label list or compared two candidates, so a test can hold +/// `topk` to the work it should not do rather than to the result, which every shape produces. +/// Test-only: a decorated sort and a partial selection select the same candidates. +#[cfg(test)] +pub(crate) mod calls { + use std::cell::Cell; + + thread_local! { + /// Label lists [`super::labels_with_name`] built on this thread. + static LABEL_LISTS: Cell = const { Cell::new(0) }; + /// Candidate pairs [`super::rank`] looked at on this thread. + static RANKS: Cell = const { Cell::new(0) }; + } + + /// Records one label list. + pub(crate) fn record_label_list() { + LABEL_LISTS.with(|lists| lists.set(lists.get() + 1)); + } + + /// Records one comparison of two candidates. + pub(crate) fn record_rank() { + RANKS.with(|ranks| ranks.set(ranks.get() + 1)); + } + + /// The label lists built on this thread since the last call, which starts a new count. + pub(crate) fn take_label_lists() -> usize { + LABEL_LISTS.with(|lists| lists.replace(0)) + } + + /// The candidate comparisons made on this thread since the last call, which starts a new count. + pub(crate) fn take_ranks() -> usize { + RANKS.with(|ranks| ranks.replace(0)) + } +} diff --git a/src/promql/src/select/tests.rs b/src/promql/src/select/tests.rs index 360d4d92445..8e739a9563c 100644 --- a/src/promql/src/select/tests.rs +++ b/src/promql/src/select/tests.rs @@ -24,7 +24,7 @@ use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis}; use datatypes::series::{Label, SeriesLabels}; use super::{ - Grouping, LabelMatcher, MatchOp, SortOrder, absent, absent_labels, limit_ratio, limitk, + Grouping, LabelMatcher, MatchOp, SortOrder, absent, absent_labels, calls, limit_ratio, limitk, natural_less, sample_offset, scalar, sort_instant, topk, }; use crate::golden::fold_pieces; @@ -33,7 +33,7 @@ use crate::grid_series::reference::{ run_ranges, sparse_series, }; use crate::grid_series::{GridSample, GridSeriesBuilder}; -use crate::histogram_eval::float_histogram::FloatHistogram; +use crate::histogram_eval::float_histogram::{FloatHistogram, copies}; fn schema(points: u64, fields: usize) -> SeriesSchema { SeriesSchema { @@ -498,6 +498,148 @@ fn limit_operators_keep_the_start_timestamp_of_a_selected_sample() { ); } +/// The selections that only move samples hand them from the field to the builder: no owned +/// histogram is built on the way, and the payload arrives as it was read. +#[test] +fn limit_operators_pass_histograms_through_without_an_owned_form() { + let histogram = + OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///"); + let schema = schema(3, 1); + let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone())).unwrap(); + builder + .push_refs(0, &[SampleRef::Histogram(histogram.view())]) + .unwrap(); + builder + .push_refs(2, &[SampleRef::Histogram(histogram.view())]) + .unwrap(); + let series = builder.finish(metric("__name__=a")).unwrap().unwrap(); + + copies::take_materialized(); + let by_ratio = limit_ratio(1.0, &schema, std::slice::from_ref(&series)).unwrap(); + let by_count = limitk( + 1.0, + &Grouping::By(Vec::new()), + &schema, + std::slice::from_ref(&series), + ) + .unwrap(); + assert_eq!( + copies::take_materialized(), + 0, + "a selection built an owned histogram of a sample it only moved" + ); + + // Both selections hand on the samples they kept, unchanged and in the same place. + let kept = |series: &Series| -> Vec<(usize, bool)> { + let samples = &series.fields.as_slice()[0]; + series + .time + .positions() + .map(|(point, row)| { + ( + point, + matches!(samples.sample(row), SampleRef::Histogram(_)), + ) + }) + .collect() + }; + for kept_series in [&by_ratio, &by_count] { + assert_eq!(kept_series.len(), 1); + assert_eq!(kept(&kept_series[0]), [(0, true), (2, true)]); + } +} + +/// `topk` builds one label list per series and ranks each point with a partial selection: the +/// comparisons it makes grow with the candidates rather than with their logarithm, which is what +/// ordering the whole group at every point cost. +#[test] +fn topk_ranks_a_point_with_a_partial_selection() { + let schema = schema(1, 1); + // Every series is one group, and the input arrives in reverse label order, so nothing about it + // is inherited from the order the members were built in. + let series: Vec = (0..256) + .rev() + .map(|index| { + let value = if index < 16 { 10.0 } else { 1.0 }; + build( + &schema, + metric(&format!("__name__=s{index:03}")), + &vec![vec![Some(value)]], + ) + .unwrap() + }) + .collect(); + + calls::take_label_lists(); + calls::take_ranks(); + let kept = topk(8.0, true, &Grouping::By(Vec::new()), &schema, &series).unwrap(); + let lists = calls::take_label_lists(); + let comparisons = calls::take_ranks(); + + let mut kept = rendered(&kept); + kept.sort(); + assert_eq!( + kept, + [ + "__name__=s000", + "__name__=s001", + "__name__=s002", + "__name__=s003", + "__name__=s004", + "__name__=s005", + "__name__=s006", + "__name__=s007" + ], + ); + assert_eq!( + lists, + series.len(), + "topk rebuilt a series' label list instead of reading the one it built" + ); + // Ordering the whole group at the point would take the candidates' logarithm per candidate; + // the partial selection takes a small constant. + assert!( + comparisons <= 4 * series.len(), + "topk compared {comparisons} candidate pairs to select 8 of {}", + series.len() + ); +} + +/// Ties go to the label set that sorts first, whatever order the input arrived in: the value the +/// partial selection stops at falls inside a run of equal values, and the run is resolved by label +/// order rather than by the order the ranking left the group in. +#[test] +fn topk_breaks_ties_by_label_order() { + let names: Vec = (0..40) + .map(|index| format!("__name__=s{index:02}")) + .collect(); + // The ten series that sort first carry the top value, and the input arrives in reverse label + // order, so a tie can only be resolved by the labels. + let values: Vec<(&str, Option)> = names + .iter() + .rev() + .map(|name| { + let value = if name.as_str() < "__name__=s10" { + 7.0 + } else { + 3.0 + }; + (name.as_str(), Some(value)) + }) + .collect(); + let (schema, series) = instant(&values); + let kept = topk(3.0, true, &Grouping::By(Vec::new()), &schema, &series).unwrap(); + let mut kept = rendered(&kept); + kept.sort(); + assert_eq!(kept, ["__name__=s00", "__name__=s01", "__name__=s02"]); + + // `bottomk` ties the same way, and its run of equal values holds the whole group. + let kept = topk(3.0, false, &Grouping::By(Vec::new()), &schema, &series).unwrap(); + let mut kept = rendered(&kept); + kept.sort(); + assert_eq!(kept, ["__name__=s10", "__name__=s11", "__name__=s12"]); +} + fn grouping(kind: &str, labels: &str) -> Grouping { match kind { "without" => Grouping::Without(names(labels)), diff --git a/src/promql/src/vector_match.rs b/src/promql/src/vector_match.rs index f1c9e0db5a6..d31e64db155 100644 --- a/src/promql/src/vector_match.rs +++ b/src/promql/src/vector_match.rs @@ -70,6 +70,33 @@ pub(crate) fn visited_pairs() -> usize { VISITED_PAIRS.with(std::cell::Cell::get) } +// How many matches the binary kernel recorded in this thread's match set. +// +// Test-only, and per thread: whether a field needs the set at all is what decides the per-sample +// work of the default one-to-one walk, and that is not visible in the result. A thread of its own +// keeps one test's matches out of another test's figure, and the counter is compiled out of the +// kernel otherwise. +#[cfg(test)] +thread_local! { + static MATCH_RECORDS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Resets the match-set counter of this thread to zero. +#[cfg(test)] +pub(crate) fn reset_match_records() { + MATCH_RECORDS.with(|recorded| recorded.set(0)); +} + +/// The matches the binary kernel has recorded in this thread's match set since the last +/// [`reset_match_records`]. +/// +/// A match is recorded once per row the operator keeps, so the count is what the walk does rather +/// than what its operands' sizes imply it would. +#[cfg(test)] +pub(crate) fn match_records() -> usize { + MATCH_RECORDS.with(std::cell::Cell::get) +} + /// A PromQL binary operator between two instant vectors. #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Hash)] pub enum BinaryOp { @@ -868,6 +895,21 @@ fn binary_field( } let drop_name = !matching.op.is_comparison() || matching.return_bool; + // One-to-one: a match group may be matched once per point; Prometheus counts a filtered + // comparison as a match too, but not an operation that failed. Filling the one side consults the + // same set. A group whose many side holds one series is matched at a point by one pairing only -- + // a second row there would have to come from a second one-side series, and two members of one + // group present at one point is the duplicate `overlap` already reports -- so the set is built + // only for the fields that can hold two matches of one group at one point, and the walk of the + // default one-to-one shape records nothing at all. + let matched = fill.lhs.is_some() + || (matching.cardinality == Cardinality::OneToOne + && shared_match_group( + many, + &matching.labels, + metadata.tables.saturating_add(metadata.pairing), + budget.as_deref_mut(), + )?); let mut results = Results::default(); let mut pairs = Pairs { matching, @@ -875,11 +917,7 @@ fn binary_field( annotations, group_keys: Vec::new(), group_ids: HashMap::new(), - // One-to-one: a match group may be matched once per point; Prometheus counts a filtered - // comparison as a match too, but not an operation that failed. Filling the many side - // consults the same set. - matched: (matching.cardinality == Cardinality::OneToOne || fill.lhs.is_some()) - .then(HashSet::new), + matched: matched.then(HashSet::new), implicit_many_to_one: None, }; for (many_index, many_series) in many.series.iter().enumerate() { @@ -1109,6 +1147,96 @@ fn binary_field( Ok(results) } +/// Whether two series of `operand` share a match group. +/// +/// The keys are compared by a hash of the labels they keep: two series that share a group hash +/// equal, and two that differ may still collide -- an answer of `true` for a group that no two +/// series share only makes the caller build the set a shared group would need. One `u64` per series +/// is sorted in place and dropped with the check, so a field of single-member groups builds no key +/// and keeps none, whatever the length of the grid. +fn shared_match_group( + operand: &Operand<'_>, + labels: &MatchingLabels, + base: usize, + mut budget: Option<&mut Budget<'_>>, +) -> Result { + if operand.series.len() < 2 { + return Ok(false); + } + if let Some(budget) = budget.as_deref_mut() { + // One hash per series, entered beside the field's own tables before the list is allocated. + budget.phase( + base.saturating_add( + operand + .series + .len() + .saturating_mul(std::mem::size_of::()), + ), + )?; + } + let mut hashes: Vec = Vec::with_capacity(operand.series.len()); + hashes.extend( + operand + .series + .iter() + .map(|series| match_group_hash(&series.metric, labels)), + ); + hashes.sort_unstable(); + let shared = hashes.windows(2).any(|pair| pair[0] == pair[1]); + // The list is dropped with the check, so the field's own phase figure is live from here on. + if let Some(budget) = budget { + budget.phase(base)?; + } + Ok(shared) +} + +/// The FNV-1a offset basis and prime, which hash one label's name and value in the same steps +/// whatever hasher the tables of a field happen to use. +const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +/// The hash of the labels [`group_key`] would keep for `metric`. +/// +/// Each kept label hashes its name and value as one string, and the labels are summed with +/// wrapping addition, so the order the label set stores them in cannot change the answer: two +/// series of one match group hash equal however they are laid out. +fn match_group_hash(metric: &SeriesLabels, labels: &MatchingLabels) -> u64 { + let label = |name: &str, value: &str| { + let mut hash = FNV_OFFSET_BASIS; + for byte in name + .as_bytes() + .iter() + .chain(&[0xff]) + .chain(value.as_bytes()) + { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(FNV_PRIME); + } + hash + }; + let mut hash = 0u64; + match labels { + MatchingLabels::On(names) => { + for name in names { + // An empty value is absent, as in `group_key`. + if let Some(value) = metric.get(name) + && !value.is_empty() + { + hash = hash.wrapping_add(label(name, value)); + } + } + } + MatchingLabels::Ignoring(ignored) => { + for tag in metric.tags() { + if !tag.value.is_empty() && !ignored.contains(&tag.name) { + hash = hash.wrapping_add(label(&tag.name, &tag.value)); + } + } + } + } + hash +} + /// The matched pairs of one field and the state Prometheus checks them against. /// /// The results are filed beside it, not in it: a pairing's open row borrows the table it files into @@ -1120,8 +1248,10 @@ struct Pairs<'a> { group_keys: Vec, group_ids: HashMap, /// The match groups an operation matched at a point, kept where Prometheus consults them: - /// one-to-one matching, and filling the many side. A filtered comparison counts as a match, - /// an operation that failed does not. + /// one-to-one matching, and filling the one side. A filtered comparison counts as a match, an + /// operation that failed does not. Kept only for the fields that need it: one whose groups hold + /// one many-side series cannot match a group twice at a point, and one filling neither side never + /// asks. matched: Option>, implicit_many_to_one: Option<(usize, usize)>, } @@ -1195,6 +1325,11 @@ impl Pairs<'_> { // One-to-one: a match group may be matched once per point; Prometheus counts a filtered // comparison as a match too, but not an operation that failed. if let Some(matched) = &mut self.matched { + // Matches are what the set is for, and it exists only for the fields that can hold two + // of one group at one point: the test-only counter reports that the walk of a + // single-member group records nothing. + #[cfg(test)] + MATCH_RECORDS.with(|recorded| recorded.set(recorded.get() + 1)); // The match set is the one container this walk grows point by point, so it is charged by // the block it takes when it outgrows what it holds -- before the insert allocates it, and // never per point of a set that has room. diff --git a/src/promql/src/vector_match/tests.rs b/src/promql/src/vector_match/tests.rs index b0139ccb698..62852393b4d 100644 --- a/src/promql/src/vector_match/tests.rs +++ b/src/promql/src/vector_match/tests.rs @@ -27,8 +27,9 @@ use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use super::{ - BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, SetOp, binary_vectors, - binary_vectors_with_reservation, set_vectors, set_vectors_with_reservation, + BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, RESULT_ROW_BYTES, SetOp, + binary_vectors, binary_vectors_with_reservation, match_records, reset_match_records, + set_vectors, set_vectors_with_reservation, }; use crate::grid_series::reference::{ Dense, OwnedHistogram, Points, assert_histogram_close, assert_result_eq, golden_cases, @@ -1199,6 +1200,163 @@ fn colliding_reset_hints_do_not_count_as_a_match() { } } +/// A group whose many side holds one series can be matched at a point by one pairing only: a second +/// row for that point would have to come from a second one-side series, and two members of one group +/// present at one point is the duplicate the one side's own scan reports. So the default one-to-one +/// shape records no match and builds no set, and the pairing still answers every point. +#[test] +fn a_single_series_group_records_no_match() { + const POINTS: usize = 1_000; + let grid = budget_grid(POINTS); + let schema = Arc::new(schema(1, &grid)); + let at: Vec = (0..POINTS).collect(); + // One series of the many side and one of the one side, in the same match group. + let lhs = vec![budget_operand(&schema, "a", &at)]; + let rhs = vec![budget_operand(&schema, "b", &at)]; + let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())); + reset_match_records(); + let output = + binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("the one pair matches"); + assert_eq!( + match_records(), + 0, + "a group of one many-side series recorded a match" + ); + // Both operands hold the point's own position, so the sum doubles it at every point. + assert_eq!(output.len(), 1, "{output:?}"); + let mut cursor = output[0].fields.as_slice()[0].cursor(); + let mut rows = 0; + for (point, row) in output[0].time.positions() { + match cursor.at(row).expect("the axis and the fields are aligned") { + SampleRef::Float(value) => assert_eq!(value, 2.0 * point as f64, "point {point}"), + _ => panic!("`+` of two floats keeps a float"), + } + rows += 1; + } + assert_eq!(rows, POINTS, "the pair answers every point"); +} + +/// A group of two many-side series can be matched twice at one point, so the set is built and the +/// second match of the group at that point is the duplicate Prometheus reports. +#[test] +fn a_group_of_two_many_series_records_its_matches() { + let grid = budget_grid(1); + let schema = Arc::new(schema(1, &grid)); + // Two left-hand series in one match group and one right-hand series they both pair with. + let lhs = [ + jobbed_operand(&schema, "a", "x", &[0]), + jobbed_operand(&schema, "b", "x", &[0]), + ]; + let rhs = [jobbed_operand(&schema, "c", "x", &[0])]; + let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())); + reset_match_records(); + let error = binary_vectors(&matching, &schema, &lhs, &schema, &rhs) + .unwrap_err() + .to_string(); + assert!( + error.contains("many-to-one matching must be explicit"), + "{error}" + ); + assert!( + match_records() >= 2, + "the second match of the group was not recorded: {}", + match_records() + ); +} + +/// Two one-side series of a group that are never present at the same point are no duplicate: each +/// point is matched by one pairing, which is what a group of one many-side series allows. +#[test] +fn a_group_matched_at_different_points_is_no_duplicate() { + const POINTS: usize = 64; + let grid = budget_grid(POINTS); + let schema = Arc::new(schema(1, &grid)); + let operand = |name: &str, instance: &str, value: f64, at: &[usize]| -> Series { + let labels = Labels::from([ + ("__name__".to_string(), name.to_string()), + ("job".to_string(), "x".to_string()), + ("instance".to_string(), instance.to_string()), + ]); + let points: Points = at + .iter() + .map(|&point| (point, f64::to_bits(value))) + .collect(); + sparse_series(&schema, &labels, &points).expect("the operand holds a value") + }; + let even: Vec = (0..POINTS).step_by(2).collect(); + let odd: Vec = (1..POINTS).step_by(2).collect(); + // One many-side series holding every point, and two one-side series that split the grid between + // them: the group is matched at every point, never twice at one. + let lhs = [operand("a", "many", 0.5, &(0..POINTS).collect::>())]; + let rhs = [ + operand("b", "even", 1.0, &even), + operand("c", "odd", 100.0, &odd), + ]; + let matching = budget_matching(BinaryOp::Add, MatchingLabels::On(vec![Arc::from("job")])); + reset_match_records(); + let output = + binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("no point is matched twice"); + assert_eq!( + match_records(), + 0, + "a group of one many-side series recorded a match" + ); + // One-to-one matching names the result after the many side and the matching labels, so both + // pairings file under one label set: the series holds the even points a `b` kept and the odd ones + // a `c` kept. + assert_eq!(output.len(), 1, "{output:?}"); + let mut cursor = output[0].fields.as_slice()[0].cursor(); + let mut rows = 0; + for (point, row) in output[0].time.positions() { + let expected = if point % 2 == 0 { 1.5 } else { 100.5 }; + match cursor.at(row).expect("the axis and the fields are aligned") { + SampleRef::Float(value) => assert_eq!(value, expected, "point {point}"), + _ => panic!("`+` of two floats keeps a float"), + } + rows += 1; + } + assert_eq!(rows, POINTS, "the two pairings cover the grid between them"); +} + +/// A fill value of the one side is answered from the matches the walk recorded, so a pairing that +/// holds at a point keeps it and only the points it left unmatched are filled -- the query of an +/// arbitrary point of a group, not a run of them. +#[test] +fn a_fill_consults_the_match_set_point_by_point() { + let grid = budget_grid(2); + let schema = Arc::new(schema(1, &grid)); + let lhs = [jobbed_operand(&schema, "a", "x", &[0])]; + let rhs = [jobbed_operand(&schema, "b", "x", &[0, 1])]; + let matching = BinaryMatching { + fill: FillValues { + lhs: Some(1_000.0), + rhs: None, + }, + ..budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())) + }; + reset_match_records(); + let output = + binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("the operands are readable"); + // Filling the one side is what asks the set whether a point was matched, so the field builds it: + // the pair at point 0 records its match, and so does the row point 1 is filled with. + assert!( + match_records() > 0, + "filling the one side did not record the match it consults" + ); + assert_eq!(output.len(), 1, "{output:?}"); + let mut cursor = output[0].fields.as_slice()[0].cursor(); + let mut kept = Vec::new(); + for (point, row) in output[0].time.positions() { + match cursor.at(row).expect("the axis and the fields are aligned") { + // Point 0 is the pair's own sum; at point 1 the fill stands in for the sample the left + // side has at no point, and the operator is applied to it as it would be to a sample. + SampleRef::Float(value) => kept.push((point, value)), + _ => panic!("`+` of two floats keeps a float"), + } + } + assert_eq!(kept, [(0, 0.0), (1, 1_001.0)], "{output:?}"); +} + /// A pair whose operation fails is not a match, so `fill_left` still fills for the right-hand /// sample of that pair; the fixtures hold floats only. #[test] @@ -2457,6 +2615,43 @@ fn a_dense_result_is_refused_where_a_sparse_one_is_admitted() { ); } +/// A single field's rows reach the builder as they were filed: no table of triples is keyed by the +/// label set the rows already sit under, so what the merge adds to the rows the field holds is the +/// columns it writes them into, and a second copy of every sample is not part of the figure. +#[test] +fn a_single_field_holds_no_second_copy_of_its_rows() { + const POINTS: usize = 8_192; + let grid = budget_grid(POINTS); + let schema = Arc::new(schema(1, &grid)); + let at: Vec = (0..POINTS).collect(); + let lhs = vec![budget_operand(&schema, "a", &at)]; + let rhs = vec![budget_operand(&schema, "b", &at)]; + let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())); + let (output, seen) = budgeted(&matching, &schema, &lhs, &schema, &rhs, usize::MAX); + let output = output.expect("the one pair matches"); + assert_eq!(output.len(), 1, "{output:?}"); + assert_eq!( + run_ranges(&output[0]).len(), + 1, + "the pair keeps one run of the whole grid" + ); + let peak = *seen + .iter() + .max() + .expect("the operation asks for its workspace"); + assert!(peak > 0, "the operation asks for its workspace"); + // What the operation holds once the field's tables are released is the rows it filed and the + // builder's own buffers: a merge that copied every row into a triple of its own table would hold + // a second copy of every sample beside them. + let floor = retained_total(&seen); + let rows = POINTS * RESULT_ROW_BYTES; + assert!(floor >= rows, "floor {floor}, rows {rows}"); + assert!( + floor <= rows + 8 * 1024, + "floor {floor} for {rows} row bytes: the merge holds a second copy of every sample" + ); +} + /// A field that meets its first histogram after a float backfills the index column of the rows /// before it, and the budget covers the column all the same. #[test] diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml index 950f5da9e82..ea60ef78bd9 100644 --- a/src/query/Cargo.toml +++ b/src/query/Cargo.toml @@ -72,6 +72,7 @@ session.workspace = true snafu.workspace = true sql.workspace = true sqlparser.workspace = true +stacker.workspace = true store-api.workspace = true substrait.workspace = true table.workspace = true diff --git a/src/query/src/metrics.rs b/src/query/src/metrics.rs index 3c0043dcc09..48f1937b62c 100644 --- a/src/query/src/metrics.rs +++ b/src/query/src/metrics.rs @@ -126,6 +126,36 @@ lazy_static! { &["result"] ) .unwrap(); + + /// Statements the native PromQL route handed to the existing planner, by the cause it recorded. + /// + /// A declined statement is not an error and not a different answer: the existing planner + /// computes the same rows, so a fallback is only visible as a slower query. This counter and + /// the log line beside it are what make its rate observable per cause. + pub static ref NATIVE_PROMQL_DECLINED_TOTAL: IntCounterVec = register_int_counter_vec!( + "greptime_query_native_promql_declined_total", + "native PromQL expressions planned by the existing planner, by cause", + &["reason"] + ) + .unwrap(); + + /// Statements whose metadata drop the native PromQL planner collected for the scan's key + /// columns alone, by reason. + /// + /// The canonical pass collects the pieces of one label set before a consumer that reads whole + /// series, and it decides that from the labels a tree reports and from the key columns the scan + /// below them keys its series by. The second half is what one pair of stored series needs: a + /// primary-key column that holds `NULL` in one series and `''` in another decodes to an absent + /// and to an empty label, and a name-dropping kernel drops the empty one with the metadata + /// labels, leaving both under one label set. A drop the labels alone prove is collected for that + /// pair, and this counter is what makes the statements that pay for it observable. + pub static ref DUPLICATE_LABELSET_CHECK_SKIPPED_TOTAL: IntCounterVec = + register_int_counter_vec!( + "greptime_query_duplicate_labelset_check_skipped_total", + "native PromQL statements whose metadata drop was collected for their key columns alone, by reason", + &["reason"] + ) + .unwrap(); } /// A stream to call the callback once a RecordBatch stream is done. diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index 25c3a5352e8..c21b17e65f8 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -22,7 +22,10 @@ pub use native::{NATIVE_PROMQL_EXTENSION_KEY, discovers_metric_name, native_prom mod set_operator; use std::collections::{BTreeSet, HashSet, VecDeque}; +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use arrow::datatypes::IntervalDayTime; use async_recursion::async_recursion; @@ -217,6 +220,45 @@ struct PromPlannerContext { range_fold_offset: Option, } +/// Stack one planning descent level needs below it before the recursion grows its stack. +/// +/// The planner recurses once per operator and the descent is a chain of nested `poll`s on boxed +/// futures, so each level's frame is charged to the stack of the level above it. A debug build's +/// frame is large: a chain of eight constant multiplications under a propagated matcher needs more +/// than the 2 MiB a worker thread has, and there is no depth limit on the expressions this route +/// accepts. The recursion grows its stack instead, at the same reserve the native runtime's own +/// walks use. +const PLANNER_STACK_RED_ZONE: usize = 512 * 1024; + +/// Stack a growth adds when the reserve above is spent. It holds several levels of the descent. +const PLANNER_STACK_GROWTH: usize = 4 * 1024 * 1024; + +/// Polls one planning descent on a stack that grows on demand. +/// +/// `stacker::maybe_grow` needs a synchronous callee to run on the segment it switches to, which an +/// `async` body cannot be, so the growth wraps a `poll` instead: the segment stays live for that +/// whole poll, every nested level the poll reaches included, and each nested level wraps its own +/// poll in turn. This is the same reserve-and-grow rule `promql::native::execute` applies to the +/// runtime node walks, which recurse synchronously. +struct GrownStack(Pin>); + +impl GrownStack { + fn new(future: F) -> Self { + Self(Box::pin(future)) + } +} + +impl Future for GrownStack { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut inner = self.get_mut().0.as_mut(); + stacker::maybe_grow(PLANNER_STACK_RED_ZONE, PLANNER_STACK_GROWTH, || { + inner.as_mut().poll(cx) + }) + } +} + /// Result labels a vector-vector binary operation derives from its matching modifier, projected /// from the operand each label belongs to. #[derive(Debug)] @@ -304,6 +346,14 @@ pub struct PromPlanner { /// The engine-wide `QueryOptions::experimental_native_promql` is applied while `QueryEngineState` /// is built; this planner reads the per-query request extension instead. native_fused: bool, + /// Whether the binary node being planned may still offer its own subtree to the native route. + /// + /// An offer compiles a whole tree, so it is made where it covers the most: the statement root, + /// and the topmost binary below a node this route does not answer. Once a binary node has spent + /// the offer, the operands below it are planned by the existing path without offering + /// themselves again - re-offering each operand of a refused chain of n operators compiles its + /// subtrees again and costs O(n^2) passes instead of O(n). + native_binary_offer: bool, } type BinaryFieldPair<'a> = (&'a String, &'a String); @@ -336,6 +386,9 @@ impl PromPlanner { ctx: PromPlannerContext::from_eval_stmt(stmt)?, promql_annotations, native_fused, + // The statement's own expression is offered below before anything else is planned, so + // the first binary node reached still holds the offer a wrapper would have spent. + native_binary_offer: true, }; // The whole statement is offered to the native compiler first, because that compiler folds @@ -380,6 +433,32 @@ impl PromPlanner { prom_expr: &PromExpr, timestamp_fn: bool, query_engine_state: &QueryEngineState, + ) -> Result { + // Every operator of the existing expression planning descends through here, so this is + // where that recursion grows its stack instead of spending the caller's, below the reserve + // [`PLANNER_STACK_RED_ZONE`] names. Two limits come with it: + // + // - The native fused-series route is no part of this recursion. Its own composition + // recursion grows at its own boundary (`PromPlanner::try_plan_native` in `native.rs`), + // not here. + // - The synchronous walks around it are not covered at all: the literal construction + // (`try_build_literal_expr`) and the logical row-key walk + // (`collect_row_key_tag_columns_from_plan`) in this file, and the topology walk + // (`native_raw_plan_bounds_region_reads`), the root-parenthesis traversal + // (`root_subquery`), the input-order dependency walk (`depends_on_input_order`), the + // selector collection (`collect_selectors_with_set_operations`) and the native root + // subquery compilation in `native.rs`, recurse synchronously on the stack their caller + // has. A deep enough expression or plan still costs them that stack. + let step = self.prom_expr_to_plan_step(prom_expr, timestamp_fn, query_engine_state); + GrownStack::new(step).await + } + + #[async_recursion] + async fn prom_expr_to_plan_step( + &mut self, + prom_expr: &PromExpr, + timestamp_fn: bool, + query_engine_state: &QueryEngineState, ) -> Result { // An anchored range call is step-invariant: evaluate it once, at the start of the // evaluation, and report its result at every step; see @@ -401,8 +480,18 @@ impl PromPlanner { .await? } PromExpr::Binary(expr) => { - self.prom_binary_expr_to_plan(query_engine_state, expr) - .await? + // One native offer per binary subtree: the offer at a node covers every node below + // it, because the compiler reads the whole tree at once. It is spent here, by the + // node that makes it, and handed back once the subtree is planned, so a refused + // chain of n operators makes one offer per position instead of one per operator + // and per subtree below it - O(n) instead of O(n^2). A wrapper this route cannot + // answer leaves the offer to the binary that sits under it. + let offer = std::mem::take(&mut self.native_binary_offer); + let plan = self + .prom_binary_expr_to_plan(query_engine_state, expr, offer) + .await; + self.native_binary_offer = offer; + plan? } PromExpr::Paren(ParenExpr { expr }) => { self.prom_expr_to_plan_inner(expr, timestamp_fn, query_engine_state) @@ -1186,10 +1275,13 @@ impl PromPlanner { }) } + /// Plans `binary_expr`; `offer` says whether this node may still offer its subtree to the + /// native route, which the caller spends for the subtree it is about to plan. async fn prom_binary_expr_to_plan( &mut self, query_engine_state: &QueryEngineState, binary_expr: &PromBinaryExpr, + offer: bool, ) -> Result { // promql-parser accepts fill modifiers, but Greptime does not implement the // required outer joins and missing-value substitution. Reject them before the @@ -1206,6 +1298,7 @@ impl PromPlanner { // The same whole native binary the statement root was offered can sit below a // legacy-only wrapper. A nested retry is valid only on the statement's own grid. if self.native_fused + && offer && self.ctx.start == self.ctx.stmt_start && self.ctx.end == self.ctx.stmt_end { @@ -1460,19 +1553,32 @@ impl PromPlanner { // A copied matcher belongs to the scan, not to the operand's identity: // `absent()` turns `selector_matcher` into the labels it reports, so the // re-planned context keeps the matchers the operand was written with. + // + // A rewritten operand is an expression this node's offer never compiled: that + // offer read the statement as it was written, and the matcher reached the scan + // afterwards. The re-plan is therefore handed the offer, which the rewritten + // operand then spends on its own subtree exactly as this node spent it - so a + // chain below the rewrite still costs one offer, not one per operator and not + // one per operator and subtree below it. if rewritten.lhs.as_ref() != lhs.as_ref() { let selectors = std::mem::take(&mut left_context.selector_matcher); - left_input = self + let offer = std::mem::replace(&mut self.native_binary_offer, true); + let replanned = self .prom_expr_to_plan(&rewritten.lhs, query_engine_state) - .await?; + .await; + self.native_binary_offer = offer; + left_input = replanned?; left_context = self.ctx.clone(); left_context.selector_matcher = selectors; } if rewritten.rhs.as_ref() != rhs.as_ref() { let selectors = std::mem::take(&mut right_context.selector_matcher); - right_input = self + let offer = std::mem::replace(&mut self.native_binary_offer, true); + let replanned = self .prom_expr_to_plan(&rewritten.rhs, query_engine_state) - .await?; + .await; + self.native_binary_offer = offer; + right_input = replanned?; right_context = self.ctx.clone(); right_context.selector_matcher = selectors; } diff --git a/src/query/src/promql/planner/native.rs b/src/query/src/promql/planner/native.rs index ad0043bf5d2..a0bec6ff492 100644 --- a/src/query/src/promql/planner/native.rs +++ b/src/query/src/promql/planner/native.rs @@ -39,6 +39,7 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE, MITO2_ENGINE}; use common_query::native_histogram::native_histogram_value_type; use common_query::prelude::OTLP_AGGREGATION_TEMPORALITY_LABEL; use common_query::series_frame::raw_series_frame_schema_with_ordering; +use common_telemetry::debug; use datafusion::arrow::datatypes::DataType as ArrowDataType; use datafusion::common::{DFSchema, DFSchemaRef}; use datafusion::error::DataFusionError; @@ -54,7 +55,7 @@ use datatypes::series::time::EvalGrid; use futures::future::BoxFuture; use promql::extension_plan::{ FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec, InputLocality, - RawSeriesSource, input_locality, matching_operands_drop_key, + RawSeriesSource, input_locality, keys_survive_metadata_drop, matching_operands_drop_key, }; use promql::native::plan::{ AggregateKind, BoundAggregate, BoundInput, BoundVectorOp, NativeExpr, NativeNode, NativeTree, @@ -64,7 +65,7 @@ use promql::native::plan::{ use promql::native::segment::{ self, NATIVE_ORDERING, canonical_boundary_stages, canonical_local_prefix, contains_canonical_merge, contains_global_consumer, groups_by_metric_name, - merge_answers_histogram, region_local_segment, segment_tree, with_row_name_column, + region_local_segment, segment_tree, with_row_name_column, }; // Kept reachable here because this crate reads them through this module's own path: the dist // planner's commutativity check asks whether a fused node is per-series work, and this crate's @@ -87,7 +88,9 @@ use crate::promql::native_compile::{ InfoInput, bind_evaluated_binary, compile_selectors_with_grid, compile_statement_with_grid, info_input, statement_grid, subquery_child_grid, }; -use crate::promql::planner::{DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, PromPlanner}; +use crate::promql::planner::{ + DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, GrownStack, PromPlanner, +}; use crate::query_engine::QueryEngineState; /// The query option that turns the native fused-series route on for one query. @@ -183,6 +186,13 @@ type PreparedInput = ( // `split_series`: whether that partition splits one series across regions, which a bare scan // cannot reconcile. A partition on tags keeps every series whole in one region. bool, + // `keys_survive`: whether the resolved table keys its series by columns that always decode a + // value, read from its own primary key through `keys_survive_metadata_drop`. It is the half of + // the metadata drop's proof a tree cannot answer about itself: the drop removes every empty + // label value with the name, so a column holding `NULL` in one series and `''` in another + // leaves both under one label set. The canonical pass collects the drop when this is `false`, + // and `SKIPPED_LABELSET_CHECK_NULLABLE_KEY` counts the statements collected for that reason. + bool, ); /// A fully placed native operand and the evaluated layout its frame carries. @@ -220,6 +230,86 @@ fn native_planned( } } +thread_local! { + /// The cause the native route recorded for the expression it is about to refuse. + /// + /// A refusal is answered by four different walks - the compiler binding, the scan preparation, + /// the output-schema derivation and the segmentation - and most of those sites are plain + /// functions that can only answer `None`. The cause therefore travels beside the `None` rather + /// than through it: a refusal records itself here, and the offer that receives the `None` + /// reports it. One statement is planned on one thread, and taking the record clears it. + static NATIVE_DECLINE: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + /// How many native attempts this thread made, one per whole-expression offer. + /// + /// An attempt compiles the whole tree below it, so this count times the statement's size bounds + /// the work planning it costs. A test uses it to hold a refused chain to one attempt per + /// position instead of one per position and per operator below it, which is O(n^2). + #[cfg(test)] + static NATIVE_ATTEMPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// The cause the route last reported for a refused offer. + /// + /// The report is a log line and a counter, neither of which a test can read back; this holds + /// the same cause the two of them carry. + #[cfg(test)] + static REPORTED_DECLINE: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + /// The reason this thread's statement had its metadata drop collected for its key columns. + /// + /// The canonical pass decides that while it rewrites a tree, and the counter that reports it is + /// taken once per statement by the offer that answers the plan - so the reason travels beside + /// the tree the way a refusal travels beside the `None` it answers. + static SKIPPED_LABELSET_CHECK: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + /// The reason the counter last reported for a drop collected over a nullable key. + /// + /// The counter is not something a test can read back; this holds the same reason it carries. + #[cfg(test)] + static REPORTED_SKIPPED_LABELSET_CHECK: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +/// The cause reported when a refusal was answered without recording one. +const NATIVE_DECLINE_UNRECORDED: &str = "cause_not_recorded"; + +/// The reason a statement's metadata drop was collected: the scan that answers it keys its series by +/// a column that can hold `NULL` in one series and `''` in another, so the labels alone cannot prove +/// the drop tells two of its series apart. +const SKIPPED_LABELSET_CHECK_NULLABLE_KEY: &str = "nullable_key"; + +/// Answers the `None` that hands this expression to the existing planner, recording why. +/// +/// Declining is the normal answer to a shape this route does not answer: the existing planner +/// computes the same rows, so the site itself stays quiet. What it contributes is the cause, which +/// the offer that receives the `None` reports once. +fn declined(reason: &'static str) -> Option { + NATIVE_DECLINE.with(|record| record.set(Some(reason))); + None +} + +/// Takes the cause the last refusal recorded, if it recorded one. +fn take_native_decline() -> Option<&'static str> { + NATIVE_DECLINE.with(|record| record.replace(None)) +} + +/// Clears the attempt count and answers how many attempts this thread made since the last call. +#[cfg(test)] +pub(crate) fn take_native_attempts() -> usize { + NATIVE_ATTEMPTS.with(|attempts| attempts.replace(0)) +} + +/// Takes the cause the route last reported for a refused offer. +#[cfg(test)] +pub(crate) fn take_reported_decline() -> Option<&'static str> { + REPORTED_DECLINE.with(|reported| reported.replace(None)) +} + +/// Takes the reason the route last reported for a drop collected over a nullable key. +#[cfg(test)] +pub(crate) fn take_reported_skipped_labelset_check() -> Option<&'static str> { + REPORTED_SKIPPED_LABELSET_CHECK.with(|reported| reported.replace(None)) +} + impl PromPlanner { /// Builds the scan of the one logical table the current context is bound to. The caller has /// already preprocessed this selector's matchers and set the context to its table. @@ -236,14 +326,14 @@ impl PromPlanner { self.ctx.table_name.clone(), self.ctx.time_index_column.clone(), ) else { - return Ok(None); + return Ok(declined("selector_without_table_or_time_index")); }; let tags = self.ctx.tag_columns.clone(); let fields = self.ctx.field_columns.clone(); // A `__field__` matcher narrows the field set in a projection the existing planner builds, and the // raw series scan below this node has no equivalent: it reports every field the scan carries. if self.ctx.field_column_matcher.is_some() || fields.is_empty() { - return Ok(None); + return Ok(declined("field_selector_without_raw_series_equivalent")); } // Delta samples are increments: the existing planner sums them for `increase` and // divides by the range for `rate`, while the native tree assumes cumulative samples. @@ -253,7 +343,7 @@ impl PromPlanner { .any(|tag| tag == OTLP_AGGREGATION_TEMPORALITY_LABEL) && under_rate_or_increase { - return Ok(None); + return Ok(declined("delta_temporality_under_rate_or_increase")); } // The selector's own table scan, built by the same method the existing path uses: // the physical planner turns it into the region scan the node reads, and building a @@ -282,7 +372,7 @@ impl PromPlanner { .map(|field| source_schema.index_of(field).ok()) .collect::>>() else { - return Ok(None); + return Ok(declined("field_missing_from_source_schema")); }; let original_projection = scan.projection.as_ref(); let selected_indices = original_projection @@ -306,7 +396,7 @@ impl PromPlanner { } None => match &input { LogicalPlan::TableScan(scan) => scan.clone(), - _ => return Ok(None), + _ => return Ok(declined("selector_plan_is_not_a_table_scan")), }, }; let table = self.table_from_source(&scan.source)?; @@ -315,6 +405,18 @@ impl PromPlanner { // only collects the pieces its own producer holds, so the caller needs this fact to tell a whole-query // label-set answer from one region's share of it. let partitioned = table_meta.partition_column_names().next().is_some(); + // A kernel that drops the metadata labels drops every label whose value is empty with them, + // so two stored series whose key columns hold `NULL` and `''` leave it as one identity. This + // is the fact the drop's proof needs beside the labels - the same rule the physical + // per-series scan reads to decide whether a node may stream - and the canonical pass + // collects the drop when it does not hold. + let keys_survive = keys_survive_metadata_drop( + tags.iter().map(String::as_str), + table_meta + .primary_key_indices + .iter() + .map(|index| &table_meta.schema.column_schemas()[*index]), + ); // An engine without a raw-series scanner has no frame to give this node, and by the time the physical // planner asks, the logical plan is already this node: the refusal there is a planning error, not a // fallback. So the engine is checked here. A file-engine table is the concrete case, since its @@ -323,7 +425,7 @@ impl PromPlanner { if !supports_native_raw_series(&table_meta.engine) || (table_meta.engine == METRIC_ENGINE_NAME && !has_verified_metric_rewrite) { - return Ok(None); + return Ok(declined("engine_without_raw_series_scanner")); } // The region's raw-series preflight refuses these, and the physical layer turns that // refusal into an error, so decline this route here instead. @@ -339,7 +441,7 @@ impl PromPlanner { .get(store_api::mito_engine_options::APPEND_MODE_KEY) .is_some_and(|value| value.eq_ignore_ascii_case("true")) { - return Ok(None); + return Ok(declined("unsupported_time_index_type_or_append_mode")); } // A partition column that is not a tag splits one series across regions, and a bare scan cannot // reconcile the fragments; a raw source can, so such a selector is admitted and its input is bound per @@ -360,10 +462,10 @@ impl PromPlanner { .iter() .map(|field| Self::field_column_type(input.schema(), field)), ) else { - return Ok(None); + return Ok(declined("mixed_field_kinds")); }; if statement_kind.is_some_and(|kind| kind != field_kind) { - return Ok(None); + return Ok(declined("selectors_disagree_on_field_kind")); } *statement_kind = Some(field_kind); let series_schema = Arc::new(SeriesSchema { @@ -383,7 +485,7 @@ impl PromPlanner { if let LogicalPlan::TableScan(table_scan) = &mut input { table_scan.filters.append(&mut filters); } else { - return Ok(None); + return Ok(declined("selector_plan_is_not_a_table_scan")); } Ok(Some(( input, @@ -393,6 +495,7 @@ impl PromPlanner { tags, partitioned, split_series, + keys_survive, ))) } @@ -401,14 +504,48 @@ impl PromPlanner { prom_expr: &PromExpr, query_engine_state: &QueryEngineState, ) -> Result> { - match self + // With the route off nothing below runs, so reporting the answer as a fallback would + // describe a route the query never asked for. + if !self.native_fused { + return Ok(None); + } + let planned = self .try_plan_native(prom_expr, query_engine_state, FusedOutputFormat::PlainRows) - .await? - { + .await?; + // Whether this statement is one whose metadata drop the canonical pass collected for its + // key columns alone. The attempts below record it where they decide it; taking it here + // keeps the count one per statement and leaves nothing behind for the next one. + let skipped = SKIPPED_LABELSET_CHECK.with(|skipped| skipped.replace(None)); + match planned { // A whole statement answers a row table. A frame is what a composition reads from its // operands, so the statement root is never one. - Some(NativePlanned::Rows(plan)) => Ok(Some(plan)), - _ => Ok(None), + Some(NativePlanned::Rows(plan)) => { + if let Some(reason) = skipped { + crate::metrics::DUPLICATE_LABELSET_CHECK_SKIPPED_TOTAL + .with_label_values(&[reason]) + .inc(); + #[cfg(test)] + REPORTED_SKIPPED_LABELSET_CHECK.with(|reported| reported.set(Some(reason))); + } + Ok(Some(plan)) + } + _ => { + // A declined expression is planned by the existing path and answers the same rows, + // so without this record a route that never ran would only be visible as a slower + // query. The cause is the refusal that decided the answer; a refusal from a walk + // that records nothing reports the generic cause rather than staying silent. + let reason = take_native_decline().unwrap_or(NATIVE_DECLINE_UNRECORDED); + debug!( + "Native PromQL route declined this expression ({reason}); \ + the existing planner answers it" + ); + crate::metrics::NATIVE_PROMQL_DECLINED_TOTAL + .with_label_values(&[reason]) + .inc(); + #[cfg(test)] + REPORTED_DECLINE.with(|reported| reported.set(Some(reason))); + Ok(None) + } } } @@ -423,13 +560,26 @@ impl PromPlanner { /// /// `Box::pin` is here only because a composition and this attempt call each other, which an anonymous /// async future cannot be sized for. + /// + /// The boxed future is polled through [`GrownStack`], which is the one place this route's recursion + /// grows its stack: a composition offers each operand here, so the descent alternates between this + /// attempt and [`Self::try_plan_native_composed_binary`], and every one of those levels is charged to + /// the stack of the level above it, below the reserve `PLANNER_STACK_RED_ZONE` names. fn try_plan_native<'a>( &'a mut self, prom_expr: &'a PromExpr, query_engine_state: &'a QueryEngineState, format: FusedOutputFormat, ) -> BoxFuture<'a, Result>> { - Box::pin(async move { + // One attempt is one whole-expression offer: a test counts them to hold the cost of a + // refused chain to its own positions, and the report below counts what a query paid for. + #[cfg(test)] + NATIVE_ATTEMPTS.with(|attempts| attempts.set(attempts.get() + 1)); + // An attempt descends the tree it was offered once per operator, and the composition below + // offers each operand in turn, so the recursion grows its stack here as well as at the + // planner's own descent: `GrownStack` grows it for the whole poll of this future, which + // covers every nested attempt the poll reaches. + Box::pin(GrownStack::new(async move { let saved = self.ctx.clone(); let whole = Box::pin(self.try_plan_native_fused_with_context( prom_expr, @@ -448,7 +598,7 @@ impl PromPlanner { .await; self.ctx = saved; composed - }) + })) } /// Combines two native operands without a legacy join. @@ -465,6 +615,9 @@ impl PromPlanner { query_engine_state: &QueryEngineState, format: FusedOutputFormat, ) -> Result> { + // Neither check refuses a shape: they say this second attempt does not apply here, so they + // record no cause - the whole-expression attempt above already named the one that decided + // the answer. if !self.native_fused || self.ctx.start != self.ctx.stmt_start || self.ctx.end != self.ctx.stmt_end @@ -492,7 +645,7 @@ impl PromPlanner { .await; self.ctx = saved.clone(); let Some(NativePlanned::Frame(left)) = left? else { - return Ok(None); + return Ok(declined("left_operand_is_not_a_frame")); }; let right = self .try_plan_native( @@ -503,10 +656,10 @@ impl PromPlanner { .await; self.ctx = saved; let Some(NativePlanned::Frame(right)) = right? else { - return Ok(None); + return Ok(declined("right_operand_is_not_a_frame")); }; if left.grid.spec() != right.grid.spec() { - return Ok(None); + return Ok(declined("operand_grids_differ")); } let NativeFramePlan { plan: left_plan, @@ -538,11 +691,13 @@ impl PromPlanner { evaluated(1, &right_output), ) { Ok(tree) => tree, - Err(DataFusionError::NotImplemented(_)) => return Ok(None), + Err(DataFusionError::NotImplemented(_)) => { + return Ok(declined("no_kernel_for_the_operator_pair")); + } Err(error) => return Err(error).context(DataFusionPlanningSnafu), }; let NativeExpr::Binary { matching, .. } = &tree.root().expr else { - return Ok(None); + return Ok(declined("bound_root_is_not_the_binary")); }; let Some(root_kind) = binary_output_field_kind( left_kind, @@ -551,10 +706,10 @@ impl PromPlanner { matching.return_bool, matching.fill, ) else { - return Ok(None); + return Ok(declined("no_output_kind_for_the_pair")); }; if !native_tree_supported(&tree) { - return Ok(None); + return Ok(declined("runtime_refuses_the_composed_tree")); } // The expression names the value columns, and it is the label the node reports. let expression = expr.to_string(); @@ -565,7 +720,7 @@ impl PromPlanner { FusedOutputFormat::PlainRows => None, FusedOutputFormat::SeriesFrame => { let Some(tags) = fused_output_tags(tree.root(), &source_tags) else { - return Ok(None); + return Ok(declined("composition_has_no_output_labels")); }; Some((tree.root().output.clone(), tags)) } @@ -580,7 +735,7 @@ impl PromPlanner { root_kind, )? else { - return Ok(None); + return Ok(declined("composition_has_no_value_columns")); }; schema } @@ -588,7 +743,9 @@ impl PromPlanner { let frame = match FusedSeries::series_frame_schema(&tree) { Ok(frame) => frame, // A frame this composition cannot declare is a shape it does not answer. - Err(DataFusionError::Plan(_)) => return Ok(None), + Err(DataFusionError::Plan(_)) => { + return Ok(declined("composition_cannot_declare_a_frame")); + } Err(error) => return Err(error).context(DataFusionPlanningSnafu), }; Arc::new( @@ -609,7 +766,7 @@ impl PromPlanner { output_schema, format, ) else { - return Ok(None); + return Ok(declined("fused_node_refuses_the_composition")); }; let plan = LogicalPlan::Extension(Extension { node: Arc::new(node), @@ -639,7 +796,7 @@ impl PromPlanner { format: FusedOutputFormat, ) -> Result> { if !self.native_fused { - return Ok(None); + return Ok(declined("native_route_not_requested")); } // The expression names the value columns of every rows layout this attempt publishes, and the // node's own label follows the same text. It is the whole statement's expression even when a @@ -652,23 +809,23 @@ impl PromPlanner { if prom_expr.value_type() == ValueType::Matrix && (subquery_root.is_none() || self.ctx.start != self.ctx.end) { - return Ok(None); + return Ok(declined("matrix_root_without_a_grid")); } let Some(stmt) = self.native_statement(prom_expr) else { - return Ok(None); + return Ok(declined("statement_has_no_evaluation_grid")); }; // The subquery root's inner expression is compiled on the child grid, with the statement left // untouched: `@ start()` and `@ end()` inside it resolve against the statement's bounds. A statement // whose grid cannot be built keeps the existing plan. let child_grid = if let Some(subquery) = subquery_root { let Ok(outer) = statement_grid(&stmt) else { - return Ok(None); + return Ok(declined("subquery_outer_grid_unavailable")); }; let Some(child) = subquery_child_grid(&stmt, &outer, subquery).context(DataFusionPlanningSnafu)? else { - return Ok(None); + return Ok(declined("subquery_has_no_child_grid")); }; Some(child.grid) } else { @@ -690,12 +847,12 @@ impl PromPlanner { &mut selectors, &mut set_operations, ) { - return Ok(None); + return Ok(declined("selector_walk_refuses_the_statement")); } // The compiler decides which calls and shapes this route answers: a name it has no kernel for comes // back as `NotImplemented`, which is a decline below. let Ok(statement) = statement_grid(&stmt) else { - return Ok(None); + return Ok(declined("statement_grid_unavailable")); }; // The one grid every lowering path below binds: a subquery root is compiled on its child // grid, and the rows it publishes carry that grid, so the split, the boundary and the @@ -706,7 +863,9 @@ impl PromPlanner { compile_stmt.expr = effective_expr.clone(); let tree = match compile_statement_with_grid(&compile_stmt, Arc::clone(&grid)) { Ok(tree) => tree, - Err(DataFusionError::NotImplemented(_)) => return Ok(None), + Err(DataFusionError::NotImplemented(_)) => { + return Ok(declined("no_kernel_for_the_statement_shape")); + } Err(error) => return Err(error).context(DataFusionPlanningSnafu), }; // A frame operand is an evaluated sample root: a composition matches series, and a @@ -717,16 +876,16 @@ impl PromPlanner { NativeType::Samples(schema) if schema.eval_grid.is_some() ); if format == FusedOutputFormat::SeriesFrame && !evaluated { - return Ok(None); + return Ok(declined("frame_operand_is_not_evaluated")); } // Bind only input tags: this data-free tree has no input, and its tags belong to its // output schema. let Some(output_tags) = fused_output_tags(tree.root(), &[]) else { - return Ok(None); + return Ok(declined("data_free_root_has_no_labels")); }; let Some(values) = native_root_fields(tree.root()) else { - return Ok(None); + return Ok(declined("data_free_root_has_no_value_columns")); }; // A label named `time` is the one collision the default time column can have: the // value columns are named after the expression, so they never take that name. The @@ -817,7 +976,7 @@ impl PromPlanner { // These matchers are alternatives the one scan below the node cannot represent, so the // query keeps the plan the existing path builds for it. if !selector.matchers.or_matchers.is_empty() { - return Ok(None); + return Ok(declined("or_matchers_have_no_single_raw_scan")); } // A selector that names no metric is answered by enumerating the schema's logical metric // tables: one input per metric, each reading its own region, so every series keeps the @@ -835,7 +994,7 @@ impl PromPlanner { .preprocess_label_matchers(&ordinary_matchers, &Some(String::new())) { Ok(m) => m, - Err(_) => return Ok(None), + Err(_) => return Ok(declined("discovery_matchers_cannot_be_preprocessed")), }; let effective_schema = self .ctx @@ -846,7 +1005,7 @@ impl PromPlanner { // see never becomes a source, so its name reaches neither the plan nor an error. let scope = self.table_provider.query_ctx().metric_discovery_scope(); if matches!(scope, MetricDiscoveryScope::Unavailable) { - return Ok(None); + return Ok(declined("metric_discovery_unavailable")); } let mut metrics = self .table_provider @@ -912,7 +1071,7 @@ impl PromPlanner { // is declined for a scan this route can read, and the same refusal keeps the empty // input from claiming a field layout the matcher would have decided. if self.ctx.field_column_matcher.is_some() { - return Ok(None); + return Ok(declined("field_selector_on_an_empty_input")); } prepared.push(empty_discovered_input( self.ctx @@ -930,10 +1089,14 @@ impl PromPlanner { // makes the whole input partitioned. let mut discovered_partitioned = false; let mut discovered_split_series = false; + // Every discovered metric's own key columns decide its series, so the union + // survives the drop only when all of them do - and a metric whose keys do not + // is enough to collect the whole input. + let mut discovered_keys_survive = true; for table in metrics { self.ctx.table_name = Some(table.table_info().name.clone()); if self.setup_context().await?.is_some() { - return Ok(None); + return Ok(declined("discovered_table_has_no_data")); } let Some(( scan, @@ -943,6 +1106,7 @@ impl PromPlanner { mut tags, partitioned, split_series, + keys_survive, )) = self .prepare_one_scan( matchers.clone(), @@ -953,14 +1117,17 @@ impl PromPlanner { ) .await? else { + // A refusal here is the selector's own: `prepare_one_scan` recorded + // its cause, and naming this pass again would overwrite it. return Ok(None); }; discovered_partitioned |= partitioned; discovered_split_series |= split_series; + discovered_keys_survive &= keys_survive; // The union reads every discovered metric, so it is region-local only when // every metric it reads is. let Some(metric) = metric else { - return Ok(None); + return Ok(declined("discovered_selector_names_no_metric")); }; // The expression names no metric, so each source reports its own: the tag is // what tells two metrics with equal ordinary labels apart, and it is what a @@ -1013,7 +1180,7 @@ impl PromPlanner { } } let Some((schema, time, mut tags, single_metric)) = selector_metadata else { - return Ok(None); + return Ok(declined("discovered_selectors_disagree_on_their_layout")); }; tags.sort(); tags.dedup(); @@ -1038,16 +1205,17 @@ impl PromPlanner { tags, discovered_partitioned, discovered_split_series, + discovered_keys_survive, )); continue; } let matchers = match self.preprocess_label_matchers(&selector.matchers, &selector.name) { Ok(m) => m, - Err(_) => return Ok(None), + Err(_) => return Ok(declined("selector_matchers_cannot_be_preprocessed")), }; if self.setup_context().await?.is_some() { - return Ok(None); + return Ok(declined("selector_has_no_data")); } let Some(one) = self .prepare_one_scan( @@ -1059,6 +1227,7 @@ impl PromPlanner { ) .await? else { + // `prepare_one_scan` recorded why, above. return Ok(None); }; prepared.push(one); @@ -1068,6 +1237,7 @@ impl PromPlanner { .await; self.ctx = original_context; let Some((mut prepared, logical_ids_differ, statement_kind)) = preparation? else { + // The selector that refused recorded its own cause; this pass only carries it up. return Ok(None); }; // `absent_over_time` reads its window through a fold that keeps the metric name, so the segment that @@ -1082,7 +1252,7 @@ impl PromPlanner { .iter() .any(|(scan, _, metric, ..)| !is_empty_discovered_input(scan) && metric.is_none()) { - return Ok(None); + return Ok(declined("presence_over_a_nameless_selector")); } // Selectors over one table share one region, so the node runs there over every scan. When they read // more than one table, each scan answers from its own region and the node runs above them; see @@ -1127,12 +1297,12 @@ impl PromPlanner { prepared[index].1 = Arc::clone(&schema); } // A real input names the output's time index and, on one table, binds every scan. - let Some((_, _, metric, time, tags, _, _)) = prepared + let Some((_, _, metric, time, tags, _, _, _)) = prepared .iter() .find(|(scan, ..)| !is_empty_discovered_input(scan)) .or_else(|| prepared.first()) else { - return Ok(None); + return Ok(declined("statement_prepared_no_readable_input")); }; let (metric, time, tags) = (metric.clone(), time.clone(), tags.clone()); // The compiler binds one input per selector in the same pre-order `collect_selectors` @@ -1197,7 +1367,9 @@ impl PromPlanner { ) { Ok(v) => v, // The compiler refusing a shape is this route declining it. - Err(DataFusionError::NotImplemented(_)) => return Ok(None), + Err(DataFusionError::NotImplemented(_)) => { + return Ok(declined("no_kernel_for_the_statement_shape")); + } Err(e) => return Err(e).context(DataFusionPlanningSnafu), }; if compiled.inputs.len() != prepared.len() { @@ -1207,11 +1379,11 @@ impl PromPlanner { )) .context(DataFusionPlanningSnafu); } - for (index, (compiled_input, (scan, _, _, time, _, _, _))) in + for (index, (compiled_input, (scan, _, _, time, _, _, _, _))) in compiled.inputs.iter().zip(prepared.iter_mut()).enumerate() { if compiled_input.source_index != index { - return Ok(None); + return Ok(declined("compiled_sources_out_of_order")); } // The range this selector reads is the compiler's own: a window and an instant differ in which // side of the lookback or range their scan starts on, and an anchor can move the whole range @@ -1309,7 +1481,7 @@ impl PromPlanner { } let source_tags = prepared .iter() - .map(|(_, _, _, _, tags, _, _)| tags.clone()) + .map(|(_, _, _, _, tags, _, _, _)| tags.clone()) .collect::>(); // The complete runtime label set of each input: a scan bound to one metric names every series it // hands over, so `__name__` is one of its labels even though it is not a tag column. Rewriting the @@ -1317,7 +1489,7 @@ impl PromPlanner { // declares the name among its tags itself. let runtime_source_tags = prepared .iter() - .map(|(_, _, metric, _, tags, _, _)| { + .map(|(_, _, metric, _, tags, _, _, _)| { let mut tags = tags.clone(); if metric.is_some() && !tags.iter().any(|tag| tag == METRIC_NAME) { tags.push(METRIC_NAME.to_string()); @@ -1325,6 +1497,20 @@ impl PromPlanner { tags }) .collect::>(); + // The other half of what the canonical pass judges a dropped label by: the one metric each + // input is bound to. A scan of one table names every series it hands over, so `__name__` is + // the same for all of them and dropping it tells none of them apart; a discovered selector + // that expands into several metrics binds no single name, which is the fact that keeps + // `rate` over it collected. + let runtime_source_metrics = prepared + .iter() + .map(|(_, _, metric, _, _, _, _, _)| metric.clone()) + .collect::>(); + // The drop's other half, and the one no tree can answer about itself: whether every input + // keys its series by columns that always decode a value. One input that cannot say so - a + // nullable primary-key column, or a discovered selector whose metrics disagree - is enough + // for the pass to collect the drop rather than trust it. + let keys_survive = prepared.iter().all(|input| input.7); // Whether any input's table is split into regions along a partition column. Only the canonical // boundary claims a whole-query label-set answer, and a region-local producer cannot see the pieces // another region holds. @@ -1336,10 +1522,26 @@ impl PromPlanner { let tree = match promql::native::canonicalize::merge_label_rewrites( &compiled.tree, &runtime_source_tags, + &runtime_source_metrics, + keys_survive, ) { Ok(tree) => tree, - Err(_) => return Ok(None), + Err(_) => return Ok(declined("label_rewrites_do_not_merge")), }; + // What those collections cost: a node that drops the metadata labels is proved by its labels + // and its scan's key columns together now, so a drop the labels alone would have proved is + // collected exactly when the keys do not survive it - the pair a primary-key column can + // spell as `NULL` in one series and `''` in another. Counting the statements collected for + // that reason is what keeps the price of the fact observable. + if promql::native::plan::leaves_a_drop_to_the_storage_keys( + &tree, + &runtime_source_tags, + &runtime_source_metrics, + keys_survive, + ) { + SKIPPED_LABELSET_CHECK + .with(|skipped| skipped.set(Some(SKIPPED_LABELSET_CHECK_NULLABLE_KEY))); + } let root = tree.root(); // The placement facts the split and the segmentation below are chosen from, and the same // facts that say whether one node can run over the scans at all. @@ -1379,13 +1581,13 @@ impl PromPlanner { .collect::>(); let per_source = prepared .iter() - .map(|(_, _, metric, time_index, tags, _, _)| { + .map(|(_, _, metric, time_index, tags, _, _, _)| { (metric.clone(), time_index.clone(), tags.clone()) }) .collect::>(); let children = prepared .into_iter() - .map(|(scan, schema, metric, _, tags, _, split_series)| { + .map(|(scan, schema, metric, _, tags, _, split_series, _)| { // A discovered selector already carries its own sources: a single metric is one // `RawSeriesSource`, several are a `Union` of them, and each names the metric it // reads. Wrapping either again would nest a source in a source. @@ -1441,38 +1643,35 @@ impl PromPlanner { // A tree with no canonical merge claims no label-set answer, so it is untouched: a // per-series query on a partitioned table keeps fusing. if !spans_regions && any_partitioned && contains_canonical_merge(root) { - return Ok(None); + return Ok(declined("canonical_merge_over_partitioned_sources")); } // The kind the tree's root answers with, which every output column follows. It is not the kind the // inputs carry: `count_over_time`, `histogram_count` and `timestamp` reduce a histogram to a // float, while a selector, a histogram-answering range function and a label rewrite pass it on. + // A canonical merge hands on the samples it was given, so an answer kind crosses it unchanged: + // a histogram answer keeps its frames over the boundary, and the router merges them with the + // same kernel it merges floats with. // A shape whose kind cannot be named keeps the existing plan rather than declaring a column the // first batch might not fill. let root_kind = match statement_kind { Some(FieldKind::Histogram) => { - // A histogram answer standing on a canonical merge would run through the canonical - // boundary carrying histogram frames, a pairing nothing exercises yet, so such a - // statement keeps the existing plan. Every merge counts, not only a root one. - if merge_answers_histogram(root, FieldKind::Histogram) { - return Ok(None); - } match root.output_field_kind(Some(FieldKind::Histogram)) { Some(root_kind) => root_kind, - None => return Ok(None), + None => return Ok(declined("root_declares_no_field_kind")), } } Some(FieldKind::Float) | None => FieldKind::Float, }; let frame_metadata = if format == FusedOutputFormat::SeriesFrame { let NativeType::Samples(schema) = &root.output else { - return Ok(None); + return Ok(declined("frame_operand_is_not_samples")); }; if schema.eval_grid.is_none() { - return Ok(None); + return Ok(declined("frame_operand_has_no_grid")); } // A segment publishes frames, so it reports no name column of its own. let Some(output_tags) = fused_output_tags(root, &source_tags) else { - return Ok(None); + return Ok(declined("root_has_no_output_labels")); }; Some((root.output.clone(), output_tags)) } else { @@ -1486,20 +1685,20 @@ impl PromPlanner { if let Some(kind @ FieldKind::Histogram) = statement_kind && ((multi_input && per_source_inputs) || aggregate_reads_histogram(root, kind)) { - return Ok(None); + return Ok(declined("histogram_aggregate_needs_state_this_route_lacks")); } // The split keeps its prefix per-series, which is why it declines an operation whose // answer depends on the order its input arrives in. A node over more than one input reads // its inputs' partitions in the order they became ready, so that refusal holds there too. if multi_input && depends_on_input_order(root) { - return Ok(None); + return Ok(declined("answer_depends_on_input_order")); } // The native output schema can name only compiled value fields, the time index and the tag // columns a source declares. A discovered source declares `__name__` because the name tells two // metrics with equal labels apart (#10106); a named source does not, so a tree reading more than // one such source cannot report the distinct groups and declines. if multi_input && groups_by_metric_name(root) { - return Ok(None); + return Ok(declined("grouping_by_metric_name_over_several_sources")); } // Catch the shape refusals before choosing native execution: errors raised during physical // planning cannot fall back. This includes unsupported aggregates and `topk`, `limitk` or @@ -1507,7 +1706,7 @@ impl PromPlanner { // tree. A plan split over regions commits to the fused node without the split or a segment // above it, so it checks itself the same way before that choice. if (multi_input || per_source_inputs) && !native_tree_supported(&tree) { - return Ok(None); + return Ok(declined("runtime_refuses_the_tree")); } // A tree that ends in a canonical merge answers one series per label set, and the pieces of one // set were spread over the scan's partitions by the rewrite that created them. Lowering that @@ -1560,7 +1759,7 @@ impl PromPlanner { if root_kind != FieldKind::Float && !(root_kind == FieldKind::Histogram && histogram_state) { - return Ok(None); + return Ok(declined("split_aggregate_state_cannot_carry_the_answer")); } // The row-value layout is what the split is validated against whichever format the // caller asked for: it names one column per value the partial state accumulates, @@ -1568,16 +1767,16 @@ impl PromPlanner { let Some(value_schema) = Self::fused_root_schema(&expression, &tree, &source_tags, &time, root_kind)? else { - return Ok(None); + return Ok(declined("root_has_no_value_columns")); }; // A frame operand also needs the evaluated layout its consumers decode, which is // the metadata derived above: a request that could not name one keeps the existing // plan rather than publishing rows to a frame consumer. if format == FusedOutputFormat::SeriesFrame && frame_metadata.is_none() { - return Ok(None); + return Ok(declined("frame_operand_needs_an_evaluated_layout")); } let Some(rawinput) = children.into_iter().next() else { - return Ok(None); + return Ok(declined("split_aggregate_has_no_input")); }; let publish_grid = Arc::clone(&grid); let node = plan_split_aggregate( @@ -1659,7 +1858,7 @@ impl PromPlanner { // contribution the consumer needs. Keep that consumer on the existing plan; a tree over more // than one input is not this shape. if required_placement && !multi_input { - return Ok(None); + return Ok(declined("cross_series_consumer_over_one_scan")); } } // The tags are asked for before the fields, because the walk answers for a tree the @@ -1670,12 +1869,14 @@ impl PromPlanner { NativeType::Scalar(_) => Vec::new(), NativeType::Samples(_) => match fused_output_tags(tree.root(), &source_tags) { Some(output_tags) => with_row_name_column(tree.root(), output_tags), - None => return Ok(None), + None => return Ok(declined("root_has_no_output_labels")), }, - NativeType::AggregateState(_) => return Ok(None), + NativeType::AggregateState(_) => { + return Ok(declined("aggregate_state_is_not_a_statement_answer")); + } }; let Some(root_fields) = native_root_fields(tree.root()) else { - return Ok(None); + return Ok(declined("root_has_no_value_columns")); }; let output_schema = match format { FusedOutputFormat::PlainRows => { @@ -1814,8 +2015,18 @@ impl PromPlanner { fn empty_discovered_input(time_index: String) -> Result { let schema = empty_discovered_series_schema(); let input = empty_discovered_frame(&schema)?; - // No table stands behind this input, so no region splits it along a partition column. - Ok((input, schema, None, time_index, Vec::new(), false, false)) + // No table stands behind this input, so no region splits it along a partition column - and no + // metadata states what its keys would survive, which is the answer that collects. + Ok(( + input, + schema, + None, + time_index, + Vec::new(), + false, + false, + false, + )) } fn empty_discovered_frame(schema: &SeriesSchema) -> Result { @@ -3235,8 +3446,9 @@ mod tests { fused_output_tags(tree.root(), &columns), "a written name is a column the output publishes" ); - let collected = promql::native::canonicalize::merge_label_rewrites(&tree, &runtime) - .expect("the pass produces a legal tree"); + let collected = + promql::native::canonicalize::merge_label_rewrites(&tree, &runtime, &[], true) + .expect("the pass produces a legal tree"); assert!( matches!(collected.root().expr, NativeExpr::Merge { .. }), "a name rewrite writes an existing runtime label: {collected}" @@ -3283,16 +3495,17 @@ mod tests { ]]; let tree = compiled_test_tree(r#"label_replace(abs(x), "__name__", "same", "missing", "(.*)")"#); - let collected = promql::native::canonicalize::merge_label_rewrites(&tree, &runtime) - .expect("the pass produces a legal tree"); + let collected = + promql::native::canonicalize::merge_label_rewrites(&tree, &runtime, &[], true) + .expect("the pass produces a legal tree"); assert!( matches!(collected.root().expr, NativeExpr::Merge { .. }), "the merge sits above the name rewrite: {collected}" ); assert_eq!( - 1, + 2, collected.to_string().matches("Merge").count(), - "exactly one merge: {collected}" + "the name rewrite and the name drop below it are each collected: {collected}" ); // The rewrite writes a name over a dropped one, so the output publishes a name column. assert_eq!( @@ -3303,11 +3516,14 @@ mod tests { // A fresh ordinary destination above the same kernel is still uncollected: no scan can carry // an ordinary label it does not declare. let fresh = compiled_test_tree(r#"label_replace(abs(x), "dst", "$1", "host", "(.*)")"#); - let collected = promql::native::canonicalize::merge_label_rewrites(&fresh, &runtime) - .expect("the pass produces a legal tree"); - assert!( - !collected.to_string().contains("Merge"), - "a fresh ordinary destination does not alias: {collected}" + let collected = + promql::native::canonicalize::merge_label_rewrites(&fresh, &runtime, &[], true) + .expect("the pass produces a legal tree"); + assert_eq!( + 1, + collected.to_string().matches("Merge").count(), + "the fresh ordinary destination adds no collection of its own, and `abs` keeps the one \ + its name drop needs: {collected}" ); } diff --git a/src/query/src/promql/planner/test.rs b/src/query/src/promql/planner/test.rs index e876d6e06bf..3d092c670a4 100644 --- a/src/query/src/promql/planner/test.rs +++ b/src/query/src/promql/planner/test.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::promql::planner::native::{ + take_native_attempts, take_reported_decline, take_reported_skipped_labelset_check, +}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -521,6 +524,7 @@ async fn plan_direct_or( ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; planner .or_operator( @@ -611,6 +615,7 @@ async fn mixed_direct_or(histogram_on_left: bool) -> (PromPlanner, LogicalPlan) ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let left_context = direct_or_context("lhs", &["job", "k"], "v"); let right_context = direct_or_context("rhs", &["job", "k"], "v"); @@ -684,6 +689,7 @@ async fn mixed_aggregate_input(histograms: Vec) -> (PromPlanner }, promql_annotations: None, native_fused: false, + native_binary_offer: true, }; (planner, plan) } @@ -1371,6 +1377,66 @@ async fn native_engine_temporality_marker_provider() -> DfTableSourceProvider { ) } +/// One Mito table of one float field whose only tag is a nullable primary-key column. +/// +/// A metric-engine table is built with nullable tags, so one series of such a table can hold `NULL` +/// in the tag where another holds `''` - the pair a metadata drop normalizes into one label set, and +/// what the canonical pass collects a drop for. The statements that pay for it are counted, which +/// `a_nullable_key_statement_records_the_collection_it_pays_for` and +/// `a_rate_chain_over_a_nullable_key_pays_one_merge` read. +async fn build_test_table_provider_with_nullable_tag(table_name: &str) -> DfTableSourceProvider { + let catalog_list = MemoryCatalogManager::with_default_setup(); + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new( + "tag_0".to_string(), + ConcreteDataType::string_datatype(), + true, + ), + ColumnSchema::new( + "timestamp".to_string(), + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "field_0".to_string(), + ConcreteDataType::float64_datatype(), + true, + ), + ])); + let table_meta = TableMetaBuilder::empty() + .schema(schema) + .engine(NATIVE_ENGINE.to_string()) + .primary_key_indices(vec![0]) + .value_indices(vec![2]) + .next_column_id(1024) + .build() + .unwrap(); + let table_info = TableInfoBuilder::default() + .name(table_name.to_string()) + .meta(table_meta) + .build() + .unwrap(); + assert!( + catalog_list + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name: table_name.to_string(), + table_id: 1024, + table: EmptyTable::from_table_info(&table_info), + }) + .is_ok() + ); + DfTableSourceProvider::new( + catalog_list, + false, + QueryContext::arc(), + DummyDecoder::arc(), + false, + ) +} + #[allow(clippy::too_many_arguments)] async fn build_test_table_provider_with_options( table_name_tuples: &[(String, String)], @@ -1457,21 +1523,35 @@ async fn build_test_native_histogram_table_provider(table_name: &str) -> DfTable build_test_native_histogram_table_provider_with_marker(table_name, false).await } +/// The same table with tag columns that always decode a value. +/// +/// A metric-engine table is built with nullable tags, so a metadata drop over it is collected for the +/// `NULL`/`''` pair a drop normalizes into one label set +/// ([`a_nullable_key_column_answers_a_histogram_statement_over_a_canonical_boundary`]). This fixture +/// states the other table: one whose keys always decode a value, so the labels alone prove the drop +/// and nothing has to be collected. +async fn build_test_native_histogram_table_provider_with_stable_keys( + table_name: &str, +) -> DfTableSourceProvider { + build_test_native_histogram_tables_with_marker(&[table_name], false, true).await +} + /// Several Mito tables of native histograms, for the multi-input gates. async fn build_test_native_histogram_tables(table_names: &[&str]) -> DfTableSourceProvider { - build_test_native_histogram_tables_with_marker(table_names, false).await + build_test_native_histogram_tables_with_marker(table_names, false, false).await } async fn build_test_native_histogram_table_provider_with_marker( table_name: &str, temporality_marker: bool, ) -> DfTableSourceProvider { - build_test_native_histogram_tables_with_marker(&[table_name], temporality_marker).await + build_test_native_histogram_tables_with_marker(&[table_name], temporality_marker, false).await } async fn build_test_native_histogram_tables_with_marker( table_names: &[&str], temporality_marker: bool, + stable_keys: bool, ) -> DfTableSourceProvider { let catalog_list = MemoryCatalogManager::with_default_setup(); let mut columns = vec![ @@ -1480,10 +1560,12 @@ async fn build_test_native_histogram_tables_with_marker( ConcreteDataType::string_datatype(), false, ), + // `le` is a nullable tag on a metric-engine table, which is what a drop over it has to be + // proved against; `stable_keys` states the table whose key columns always decode a value. ColumnSchema::new( LE_COLUMN_NAME.to_string(), ConcreteDataType::string_datatype(), - true, + !stable_keys, ), ]; if temporality_marker { @@ -2695,6 +2777,33 @@ async fn native_absent_over_time_plans_over_discovered_metrics() { ); } +/// Whether any canonical merge in `plan` asks to hand histogram frames across a boundary. +/// +/// A merge hands on the samples it was given, so its answer kind is its child's - which is the fact +/// the admission gate this route used to carry read, and the reason it kept such a statement on the +/// existing plan. This reads the same fact off the plan the route now builds, so a test says +/// "histogram frames cross this boundary" about the node that carries them rather than about the +/// statement's answer kind. +fn fused_merges_answer_histograms(plan: &LogicalPlan) -> bool { + fn visit(plan: &LogicalPlan, found: &mut bool) { + if let LogicalPlan::Extension(Extension { node }) = plan + && let Some(fused) = node.as_any().downcast_ref::() + { + *found |= fused.tree().root().any(&|node| { + matches!(node.expr, NativeExpr::Merge { .. }) + && node.output_field_kind(Some(FieldKind::Histogram)) + == Some(FieldKind::Histogram) + }); + } + for input in plan.inputs() { + visit(input, found); + } + } + let mut found = false; + visit(plan, &mut found); + found +} + /// Every native tree the plan carries, outermost first: one per stage the plan splits into. fn fused_trees(plan: &LogicalPlan) -> Vec { fn visit(plan: &LogicalPlan, trees: &mut Vec) { @@ -3234,6 +3343,176 @@ async fn native_fused_rate_plans_to_a_fused_series_node() { ); } +/// A single-metric `rate` streams: the window drops the metric name, but every series the scan +/// hands over is named the same one, so the drop tells none of them apart. Nothing has to collect +/// what the window hands out, and the plan keeps the one node over the scan. +#[tokio::test] +async fn native_fused_single_metric_rate_streams_without_a_canonical_merge() { + let query = "rate(some_metric[5m])"; + let plan = build_native_fused_plan(query, true).await; + let fused = find_fused_series(&plan).expect("the native route answers the query"); + assert_eq!( + Some("some_metric"), + fused.binding().metric_name.as_deref(), + "the binding names the one metric the scan reads" + ); + assert!( + !fused.is_canonical_producer(), + "`{query}` needs no canonical boundary: {plan}" + ); + assert!( + matches!(fused.tree().root().expr, NativeExpr::Window { .. }), + "`{query}`: the root stays the window: {:?}", + fused.tree().root().expr + ); + let [child] = fused.children() else { + panic!("`{query}` reads one input: {plan}"); + }; + assert!( + matches!(child, LogicalPlan::TableScan(_)), + "`{query}`: the window reads the scan itself: {plan}" + ); +} + +/// A `rate` over a selector that discovered several metrics keeps its collection: two of those +/// metrics can agree on every ordinary label, so the name the window drops is the only thing +/// telling them apart - and the merge is what reports the collision. +#[tokio::test] +async fn native_fused_multi_metric_rate_keeps_its_canonical_merge() { + let query = r#"rate({tag_0="x"}[5m])"#; + let plan = build_native_fused_plan_with_provider( + query, + true, + metric_discovery_provider(&["alpha_total", "beta_total"]).await, + ) + .await; + let consumer = + find_fused_series(&plan).unwrap_or_else(|| panic!("`{query}` is not native: {plan}")); + let [child] = consumer.children() else { + panic!("`{query}`: the boundary has one producer: {plan}"); + }; + let producer = find_fused_series(child).unwrap_or_else(|| panic!("`{query}`: {plan}")); + assert!(producer.is_canonical_producer(), "`{query}`: {plan}"); + assert!( + matches!(producer.tree().root().expr, NativeExpr::Merge { .. }), + "`{query}`: the window's pieces are collected: {:?}", + producer.tree().root().expr + ); +} + +/// A statement whose scan keys its series by a nullable column is collected for that column, and the +/// counter says so once per statement. +/// +/// The labels half of the drop's proof holds here: `rate(some_metric[5m])` over one metric proves +/// the name it drops is the one every series of the scan shares. The other half is the scan's key +/// columns, and this fixture's tag is a nullable primary key: one series can hold `NULL` there and +/// another `''`, which decode to no label once the drop removes the empty value with the name. The +/// pass therefore collects the drop - and reads the table to say so - which is what the count makes +/// observable. +/// +/// A window that keeps the name drops nothing, so it relies on nothing and counts nothing. +#[tokio::test] +async fn a_nullable_key_statement_records_the_collection_it_pays_for() { + let table_provider = build_test_table_provider_with_nullable_tag("some_metric").await; + let plan = + build_native_fused_plan_with_provider("rate(some_metric[5m])", true, table_provider).await; + let trees = fused_trees(&plan); + assert!( + !trees.is_empty(), + "`rate(some_metric[5m])` left the native route" + ); + assert!( + trees.iter().any(|tree| tree.contains("Merge")), + "the nullable key collected the window's drop: {trees:?}" + ); + assert_eq!( + Some("nullable_key"), + take_reported_skipped_labelset_check(), + "the drop was collected for the key columns and not counted: {}", + plan.display_indent_schema() + ); + + let table_provider = build_test_table_provider_with_nullable_tag("some_metric").await; + let plan = build_native_fused_plan_with_provider( + "last_over_time(some_metric[5m])", + true, + table_provider, + ) + .await; + assert!( + find_fused_series(&plan).is_some(), + "`last_over_time(some_metric[5m])` left the native route: {}", + plan.display_indent_schema() + ); + assert_eq!( + None, + take_reported_skipped_labelset_check(), + "a window that keeps the name collects nothing for the keys: {}", + plan.display_indent_schema() + ); +} + +/// A nullable key column costs a chain of metadata drops one merge, not one per drop. +/// +/// The merge sits above the window - the bottom-most drop - and everything above it reads one series +/// per canonical label set, so the drops above it need no collection of their own. A merge per drop +/// would buffer the whole input once per kernel for a pair only the bottom one can still fold. +#[tokio::test] +async fn a_rate_chain_over_a_nullable_key_pays_one_merge() { + for query in [ + "rate(some_metric[5m]) * 2", + "abs(abs(rate(some_metric[5m])))", + ] { + let table_provider = build_test_table_provider_with_nullable_tag("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let trees = fused_trees(&plan); + assert!( + !trees.is_empty(), + "`{query}` is not native: {}", + plan.display_indent_schema() + ); + let merges: usize = trees.iter().map(|tree| tree.matches("Merge").count()).sum(); + assert_eq!( + 1, merges, + "`{query}` pays for one merge over the nullable key: {trees:?}" + ); + } +} + +/// A scan that carries `__type__` or `__unit__` as a tag can hold two series that agree on +/// everything else, and a range function drops those two along with the name. A fixed metric name +/// proves nothing about them, so the collection stays. +#[tokio::test] +async fn native_fused_metadata_tag_in_the_binding_keeps_the_collection() { + for metadata in ["__type__", "__unit__"] { + let query = "rate(some_metric[5m])"; + let specs = vec![( + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + 2, + 1, + )]; + let tags = vec![vec!["tag_0".to_string(), metadata.to_string()]]; + let plan = build_native_fused_plan_with_provider( + query, + true, + build_test_table_provider_with_tsid_tag_names(&specs, &tags, None).await, + ) + .await; + let consumer = find_fused_series(&plan) + .unwrap_or_else(|| panic!("{metadata}: `{query}` is not native: {plan}")); + let [child] = consumer.children() else { + panic!("{metadata}: `{query}`: the boundary has one producer: {plan}"); + }; + let producer = + find_fused_series(child).unwrap_or_else(|| panic!("{metadata}: `{query}`: {plan}")); + assert!( + matches!(producer.tree().root().expr, NativeExpr::Merge { .. }), + "{metadata}: `{query}` keeps the collection: {:?}", + producer.tree().root().expr + ); + } +} + #[tokio::test] async fn native_fused_time_partitioned_table_reads_through_a_raw_source() { let table_provider = native_engine_provider(vec![1]).await; @@ -3811,6 +4090,130 @@ async fn native_fused_metric_engine_two_aggregate_pair_is_one_execution() { } } +/// Every statement is offered to the native route once, and a statement the flag leaves alone is +/// offered none. +/// +/// The count is what bounds planning: one offer compiles the whole tree below it, so a route that +/// re-offered a subtree for every operator below it would cost the square of the expression's size. +#[tokio::test] +async fn native_offer_is_counted_once_per_statement() { + let _ = take_native_attempts(); + let plan = build_native_fused_plan("rate(some_metric[5m])", true).await; + assert!(find_fused_series(&plan).is_some(), "{plan}"); + assert_eq!( + 1, + take_native_attempts(), + "the statement is offered exactly once" + ); + + let plan = build_native_fused_plan("rate(some_metric[5m])", false).await; + assert!(find_fused_series(&plan).is_none(), "{plan}"); + assert_eq!( + 0, + take_native_attempts(), + "the flag off offers nothing to the native route" + ); +} + +/// A propagated matcher is offered once for the operand it rewrote, and nothing below it. +/// +/// `matching_filters::propagate` writes a matcher into an operand's scan, which makes that operand +/// an expression the offer spent at its parent never compiled: that offer read the statement as it +/// was written, and the matcher reached the scan afterwards. The rewrite is therefore offered of +/// its own, and the offer it is handed is spent again by the rewritten operand's own subtree, +/// exactly as this one was. So `N` operators below the rewrite buy one offer - not `N`, and not the +/// `N` positions times `N` subtrees an unconditional re-offer would compile. +#[tokio::test] +async fn native_offer_counts_a_propagated_operand_once() { + // The same match with the rewritten operand two operators deep and six deep: the matcher + // crosses in both, and the four extra operators are the operand's subtree rather than more + // expressions to offer. + let shallow = "((some_metric * 2) * 2) / on(tag_0) \ + some_metric{tag_0=\"x\",__field__=\"field_0\"}"; + let deep = "((((((some_metric * 2) * 2) * 2) * 2) * 2) * 2) / on(tag_0) \ + some_metric{tag_0=\"x\",__field__=\"field_0\"}"; + let mut counts = Vec::new(); + for query in [shallow, deep] { + let _ = take_native_attempts(); + let plan = build_native_fused_plan(query, true).await; + let display = plan.display_indent().to_string(); + // The count below is only a bound if the matcher really crossed to the other operand. + assert!( + display.contains("some_metric.tag_0 = Utf8(\"x\")"), + "{query}: the matcher reached the other operand's scan: {display}" + ); + counts.push(take_native_attempts()); + } + // Seven offers either way: the statement, its root's own retry under the legacy-only + // `__field__` matcher, the frame operands the composed attempts beside those two read, and the + // propagated operand itself. The four operators that make the deep statement deeper are inside + // that operand's subtree, so they buy no offer of their own - an offer that was not spent again + // by the rewrite would compile each of them, and the deep statement would cost four more. + assert_eq!( + (7, 7), + (counts[0], counts[1]), + "a propagated operand is offered once whatever its depth: seven offers at both depths" + ); +} + +/// The same matcher-propagating match, with its rewritten operand `depth` operators deep. +fn propagation_query(depth: usize) -> String { + let operand = (0..depth).fold("some_metric".to_string(), |inner, _| { + format!("({inner} * 2)") + }); + format!("{operand} / on(tag_0) some_metric{{tag_0=\"x\",__field__=\"field_0\"}}") +} + +/// How a statement of [`propagation_query`] plans: whether a native node answers the rewritten +/// operand, and whether the propagated matcher reached the other operand's scan. +async fn propagation_route(depth: usize) -> (bool, bool) { + let plan = build_native_fused_plan(&propagation_query(depth), true).await; + let display = plan.display_indent().to_string(); + ( + find_fused_series(&plan).is_some(), + display.contains("some_metric.tag_0 = Utf8(\"x\")"), + ) +} + +/// A chain under a propagated matcher plans on the stack a query worker has, however deep it is. +/// +/// The planner recurses once per operator and charges every level's frame to the stack of the level +/// above it, so a chain of constant multiplications deep enough - eight operators was enough on a +/// two-megabyte thread before the descent grew its own stack - aborted the process rather than +/// answering. nextest's own test thread is not a worker's, so the depths are planned on a thread +/// this test sizes like one. +#[test] +fn a_deep_propagated_operand_plans_within_a_worker_stack() { + let route = std::thread::Builder::new() + .name("deep_propagated_operand".to_string()) + .stack_size(2 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a current-thread runtime"); + runtime.block_on(async { + let shallow = propagation_route(2).await; + for depth in [8, 10, 12] { + assert_eq!( + shallow, + propagation_route(depth).await, + "an operand {depth} operators deep plans as one 2 operators deep does" + ); + } + shallow + }) + }) + .expect("a thread with a worker's stack") + .join() + .expect("operands 2, 8, 10 and 12 operators deep plan on a two-megabyte stack"); + assert_eq!( + (true, true), + route, + "the rewritten operand is native and the matcher reached the other operand's scan" + ); +} + /// A metric table without logical-table metadata declines to the legacy plan. #[tokio::test] async fn native_fused_metric_table_without_logical_metadata_falls_back() { @@ -3902,6 +4305,12 @@ async fn native_fused_engine_without_raw_series_falls_back() { without_flag.display_indent().to_string(), "a file-engine table must keep the existing plan" ); + // The refusal is reported with its cause, not a bare Ok(None). + assert_eq!( + take_reported_decline(), + Some("engine_without_raw_series_scanner"), + "the refusal names the engine's missing raw-series scanner" + ); } /// A value field the raw-series readers cannot serve keeps the existing plan. @@ -4907,8 +5316,10 @@ async fn native_composed_binary_declines_to_the_existing_plan() { ); // An operand that is not a frame leaves the statement's own wrapper in place: the pair is - // refused, so the existing plan answers the statement while the operand that does compose keeps - // its own answer below it. + // refused, so the existing plan answers the statement. The offer is the statement's, and the + // binary that makes it spends it for the whole subtree below: the operand is planned by the + // existing path too, because a refused chain of n operators would otherwise compile its + // subtrees again at every operator. for query in [ // A scalar root is not a frame: composing it by vector matching would broadcast it as if it // were a one-series vector, which is not what the query asked for. @@ -4916,7 +5327,7 @@ async fn native_composed_binary_declines_to_the_existing_plan() { // Neither is a scalar function, however it is written. "(hist * on(tag_0) floats) / time()", // A comparison between a histogram and a float has no kernel at this pair either, so the - // operand's own answer stops where it was admitted. + // whole statement stops where the pair was refused. "(hist * on(tag_0) floats) > on(tag_0) floats", ] { let plan = @@ -4928,8 +5339,22 @@ async fn native_composed_binary_declines_to_the_existing_plan() { "`{query}` has no composed answer: {display}" ); assert!( - find_fused_series(&plan).is_some(), - "`{query}` keeps the operand's own answer: {display}" + find_fused_series(&plan).is_none(), + "`{query}` spends its one offer above the operand: {display}" + ); + let legacy = PromPlanner::stmt_to_plan_with_annotations( + native_mixed_kind_provider().await, + &build_eval_stmt(query), + &build_query_engine_state(), + None, + false, + ) + .await + .unwrap_or_else(|error| panic!("`{query}`: the existing route must answer it: {error}")); + assert_eq!( + legacy.display_indent().to_string(), + display, + "`{query}` must keep the existing plan" ); } } @@ -5839,7 +6264,6 @@ async fn native_fused_pointwise_wrapper_around_a_pair_is_admitted() { "1 - some_metric / some_metric", "(some_metric / some_metric) * 100", "abs(some_metric - on(tag_0) some_metric)", - "(some_metric > some_metric) * 100", ] { let plan = build_native_fused_plan(query, true).await; let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}")); @@ -5853,6 +6277,44 @@ async fn native_fused_pointwise_wrapper_around_a_pair_is_admitted() { assert_eq!(2, fused.inputs().len(), "{query}"); } + // A filtering comparison keeps the metric name, so it hands its series over under a label set + // that still tells them apart; the arithmetic broadcast above it drops that name, and the drop + // is only injective when the name it removes is one value over every series below it. The label + // walk cannot prove that across a match - a match reports its operands' labels, not one + // metric's - so the broadcast is collected, and the collection turns the statement into a + // canonical boundary over one segment that reads each occurrence through its own source. + let query = "(some_metric > some_metric) * 100"; + let plan = build_native_fused_plan(query, true).await; + let consumer = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}")); + assert!( + matches!(consumer.tree().root().expr, NativeExpr::Input(_)), + "{query}: the consumer reads the collected boundary: {plan}" + ); + let [child] = consumer.children() else { + panic!("{query}: the boundary has one producer: {plan}"); + }; + let producer = find_fused_series(child).unwrap_or_else(|| panic!("{query}: {plan}")); + assert!(producer.is_canonical_producer(), "{query}: {plan}"); + assert!( + matches!(producer.tree().root().expr, NativeExpr::Merge { .. }), + "{query} collects the broadcast's name drop: {:?}", + producer.tree().root().expr + ); + // The merge collects what the two occurrences produced, so the work below the boundary is one + // segment reading each occurrence through its own source. + let [segment] = producer.children() else { + panic!("{query}: the producer reads one segment: {plan}"); + }; + let segment = find_fused_series(segment).unwrap_or_else(|| panic!("{query}: {plan}")); + assert_eq!(2, segment.inputs().len(), "{query}: {plan}"); + for input in segment.inputs() { + assert!( + matches!(input, LogicalPlan::Extension(extension) + if extension.node.as_any().is::()), + "{query}: each occurrence reads its own source: {plan}" + ); + } + let query = "ceil(some_metric)"; let plan = build_native_fused_plan(query, true).await; assert!(find_fused_series(&plan).is_some(), "{query}: {plan}"); @@ -6841,7 +7303,7 @@ async fn native_fused_limitk_over_a_windowed_aggregate_keeps_its_shape() { ); assert_eq!( Some(true), - unifies_its_own_contributions(window_node), + unifies_its_own_contributions(window_node, &[], &[]), "`{query}`: the window emits one series per label set, so `limitk` collects nothing" ); let NativeExpr::Aggregate { @@ -9541,6 +10003,7 @@ async fn comparison_binary_join_uses_tsid_and_keeps_it_in_filtered_result() { ctx: PromPlannerContext::from_eval_stmt(&eval_stmt).unwrap(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let plan = planner .prom_expr_to_plan(&eval_stmt.expr, &build_query_engine_state()) @@ -10535,7 +10998,8 @@ async fn an_aggregate_over_an_unnamed_kind_keeps_the_existing_plan() { "sum(some_metric or some_metric)", "sum(some_metric and some_metric)", ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); assert!( @@ -10561,19 +11025,30 @@ async fn an_aggregate_over_an_unnamed_kind_keeps_the_existing_plan() { pending.extend(node.inputs()); } } - // A float-producing window still permits the aggregate itself to split natively: its count, - // not the input histogram, reaches the fold. - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + // A float-producing window still answers on the route: its count, not the input histogram, + // reaches the fold. The name `count_over_time` drops is the one every series of a single-metric + // scan shares, and these tables' key columns always decode a value, so the labels are the whole + // proof and nothing collects the prefix - and a prefix nothing collects is exactly what the + // split folds: the statement runs as a region-side partial aggregate and a final fold above it. + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider( "sum(count_over_time(some_metric[5m]))", true, table_provider, ) .await; + let display = plan.display_indent_schema().to_string(); + let partial = find_fused_series(&plan) + .unwrap_or_else(|| panic!("the float prefix left the native route: {display}")); assert!( find_final_split(&plan).is_some(), - "the float prefix no longer splits natively: {}", - plan.display_indent_schema() + "the uncollected prefix did not split: {display}" + ); + assert!( + matches!(partial.tree().root().expr, NativeExpr::Aggregate { .. }), + "the state-exporting producer folds: {}\n{display}", + partial.tree() ); } @@ -10581,81 +11056,136 @@ async fn an_aggregate_over_an_unnamed_kind_keeps_the_existing_plan() { /// /// The canonicalization pass wraps a collapsed rewrite in a merge, which the kind walk used to /// leave undescribed. "Cannot name it" then read as "possibly a histogram", so -/// `label_replace(histogram_count(h), ...)` and its aggregated form kept the existing plan even -/// though both answer floats. The walk now names a merge (it concatenates samples of one field) -/// and a set operation (its sides agree on the kind), so those statements stay on the route while -/// a histogram that really does enter an aggregate still keeps the existing plan. +/// `label_replace(histogram_count(h), ...)` kept the existing plan even though it answers floats. +/// The walk now names a merge (it concatenates samples of one field) and a set operation (its sides +/// agree on the kind), so that statement stays on the route - while a fold above the collection the +/// same rewrite's input needs, and a histogram that really does enter an aggregate or stand on a +/// canonical merge, still keep the existing plan. #[tokio::test] async fn a_label_rewrite_keeps_a_float_answer_on_the_route() { - for query in [ - "label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\")", - "sum(label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\"))", + let query = "label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\")"; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let fused = find_fused_series(&plan) + .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); + // The value column is resolved by its type: the expression names it, so the compiled field + // name is not what a reader can address. + let field = native_value_field(fused, &format!("{query}: {display}")); + assert_eq!( + &ArrowDataType::Float64, + field.data_type(), + "{query} declared a histogram column for a float answer: {display}" + ); + // A histogram answer standing on the same collection crosses it: the merge carries histogram + // frames, and where the statement answers a float the map reducing them sits above the boundary. + // Both orderings are admitted because the merge only concatenates one field's samples - what it + // carries is its child's business - so each of these names the kind it publishes. + for (query, histogram) in [ + ( + "label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")", + true, + ), + ( + "histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\"))", + false, + ), + ( + "sum(histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")))", + false, + ), ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan) .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); - let field = native_value_field(fused, &format!("{query}: {display}")); assert_eq!( - &ArrowDataType::Float64, - field.data_type(), - "{query} declared a histogram column for a float answer: {display}" + if histogram { + native_histogram_value_type().as_arrow_type() + } else { + ArrowDataType::Float64 + }, + *native_value_field(fused, &format!("{query}: {display}")).data_type(), + "{query} declared the wrong column kind: {display}" ); - } - // A histogram answer standing on a canonical merge is out of this route's reach for now: the - // canonical boundary carrying histogram frames is the follow-up's subject. Both orderings - // count - the merge at the root, and a merge below a map that answers floats - because a merge - // anywhere in the tree is a boundary that would carry histograms. - for query in [ - "label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")", - "histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\"))", - "sum(histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")))", - ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let trees = fused_trees(&plan); assert!( - find_fused_series(&plan).is_none(), - "{query} took a histogram answer through the canonical boundary: {}", - plan.display_indent_schema() + trees + .iter() + .any(|tree| tree.contains("Merge: canonical label-set merge")), + "{query} answers without the merge the rewrite needs: {trees:?}" ); } + // The map inside the rewrite answers a float, so the one collection left is the rewrite's own, + // and a fold above that merge does not carry a histogram: the name `histogram_count` drops is + // the one every series of a single-metric scan shares, which the labels alone prove. The + // statement therefore stays on the route, where it was collected twice before. + let query = + "sum(label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\"))"; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let fused = find_fused_series(&plan) + .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); + let field = native_value_field(fused, &format!("{query}: {display}")); + assert_eq!( + &ArrowDataType::Float64, + field.data_type(), + "{query} declared a histogram column for a float answer: {display}" + ); } #[tokio::test] async fn a_segmented_parent_answers_the_kind_of_the_uncut_root() { - for (query, histogram) in [ - ( - "count_over_time(some_metric[5m]) + on (__name__) count_over_time(some_metric[5m])", - false, - ), - ( - "histogram_count(some_metric) + on (__name__) histogram_count(some_metric)", - false, - ), - // The same segmentation over a tree that does answer histograms keeps the histogram - // column, which is what tells the two derivations apart: a kind forced to `Float` would - // pass the two cases above and fail this one. - ("some_metric * 2 + on (__name__) some_metric * 2", true), + // Both operands answer floats, so the parent's column is the float one whatever the segments + // below it read: a kind taken from the scan's histogram field would declare the wrong column. + for query in [ + "count_over_time(some_metric[5m]) + on (__name__) count_over_time(some_metric[5m])", + "histogram_count(some_metric) + on (__name__) histogram_count(some_metric)", ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan).unwrap_or_else(|| { panic!("{query} left the native route instead of segmenting: {display}") }); let field = native_value_field(fused, &format!("{query}: {display}")); - let expected = if histogram { - native_histogram_value_type().as_arrow_type() - } else { - ArrowDataType::Float64 - }; assert_eq!( - &expected, + &ArrowDataType::Float64, field.data_type(), "{query} declared the wrong column kind: {display}" ); } + // A histogram answer through the same match stays on the route as long as nothing collects it: + // `some_metric * 2` drops the metadata labels over one metric, which the labels prove, so no + // merge carries histogram frames and the gate that refuses one is never reached. + let query = "some_metric * 2 + on (__name__) some_metric * 2"; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + assert!( + find_fused_series(&plan).is_some(), + "{query} left the native route although nothing collects it: {display}" + ); + let mut pending = vec![&plan]; + while let Some(node) = pending.pop() { + if let LogicalPlan::Extension(Extension { node }) = node + && let Some(fused) = node.as_any().downcast_ref::() + { + assert!( + !matches!(fused.tree().root().expr, NativeExpr::Merge { .. }), + "{query} collects a histogram answer: {}\n{display}", + fused.tree() + ); + } + pending.extend(node.inputs()); + } } #[tokio::test] @@ -10675,6 +11205,157 @@ async fn native_histogram_selector_plans_a_native_producer() { ); } +/// A nullable key column collects a histogram statement onto a canonical boundary. +/// +/// A metric-engine table is built with nullable tag columns, so a metadata drop over one is collected +/// for the `NULL`/`''` pair it normalizes into one label set - and for a histogram answer that +/// collection is a canonical merge carrying histogram frames. It is the boundary a float answer gets: +/// a producer running the statement's own tree per partition, the router collecting one label set +/// from every partition with the kernel that merges floats as it merges histograms, and a consumer +/// answering the histogram column above it. The same statement over a table whose key columns always +/// decode a value is proved by its labels, so nothing is collected and the histogram answer is one +/// node. +#[tokio::test] +async fn a_nullable_key_column_answers_a_histogram_statement_over_a_canonical_boundary() { + for query in ["rate(some_metric[5m])", "some_metric * 2"] { + let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let fused = find_fused_series(&plan) + .unwrap_or_else(|| panic!("`{query}` left the route instead of answering: {display}")); + assert_eq!( + &native_histogram_value_type().as_arrow_type(), + native_value_field(fused, &format!("`{query}`: {display}")).data_type(), + "`{query}` declared something other than the histogram it answers: {display}" + ); + // The merge is the collection the drop is paid for, and it answers what its child answers: + // the statement's kind, histograms included. + let trees = fused_trees(&plan); + assert!( + trees + .iter() + .any(|tree| tree.contains("Merge: canonical label-set merge")), + "`{query}` answers without the merge its drop is collected into: {trees:?}" + ); + // What crosses the boundary is the histogram payload itself: the merge that collected the + // drop answers histograms, so the frames the producer hands over are histogram frames. + assert!( + fused_merges_answer_histograms(&plan), + "`{query}` collected a field that is not the histogram it answers: {display}" + ); + } + // Over a table whose keys always decode a value the labels alone prove the drop, so the answer + // is one node and nothing collects it. + for query in ["rate(some_metric[5m])", "some_metric * 2"] { + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let trees = fused_trees(&plan); + assert!( + find_fused_series(&plan).is_some(), + "`{query}`: the proved drop left the route: {display}" + ); + assert!( + !trees.iter().any(|tree| tree.starts_with("Merge")), + "`{query}` collected a proved drop: {trees:?}" + ); + } +} + +/// A float answer reduces the histograms a boundary carried, and the boundary stays below it. +/// +/// `histogram_count(rate(h[5m]))` answers a float, so nothing about its root kind says how the +/// window below it is carried: `rate` drops the metadata labels over a nullable key, so the window +/// is collected into a canonical merge that hands histogram frames across the boundary, and +/// `histogram_count` maps those histograms to floats above it. The frames the producer publishes are +/// still histogram frames - the payload struct is what says so - and the consumer answers the float +/// column. +#[tokio::test] +async fn a_float_answer_reduces_the_histograms_a_boundary_carried() { + let query = "histogram_count(rate(some_metric[5m]))"; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let fused = find_fused_series(&plan) + .unwrap_or_else(|| panic!("`{query}` left the route instead of answering: {display}")); + assert_eq!( + &ArrowDataType::Float64, + native_value_field(fused, &format!("`{query}`: {display}")).data_type(), + "`{query}` declared a histogram column for a float answer: {display}" + ); + let trees = fused_trees(&plan); + let boundary = trees + .iter() + .find(|tree| tree.contains("Merge: canonical label-set merge")) + .unwrap_or_else(|| panic!("`{query}` answers without the collected window: {trees:?}")); + // What the merge collects is the window that dropped the name, and the field it hands over is + // the histogram payload: the native frame schema carries the histogram struct's own columns. + assert!( + boundary.contains("Window: rate "), + "`{query}`: the boundary collects something other than the window: {boundary}" + ); + // The frames the boundary hands over hold histograms, which is what says the map above it reads + // histograms rather than floats handed to it in their place. + assert!( + fused_merges_answer_histograms(&plan), + "`{query}`: the boundary carries a field that is not a histogram: {display}" + ); + // The map that answers the float reads the collected frames, so it sits above the boundary. + let reducing = trees + .iter() + .position(|tree| tree.contains("histogram_count")) + .unwrap_or_else(|| panic!("`{query}` has no map that reduces the histograms: {trees:?}")); + let collecting = trees + .iter() + .position(|tree| tree.contains("Merge: canonical label-set merge")) + .expect("the boundary was found above"); + assert!( + reducing != collecting, + "`{query}`: the reduction is the boundary itself: {trees:?}" + ); +} + +/// A histogram answer that crosses a collected drop still segments. +/// +/// The tree's two operands each drop the metadata labels over a nullable key, so each is collected, +/// and the match above them reads a histogram from each side. Segmentation cuts the tree into +/// segments that run where their inputs are read and keeps the collected boundary as a frame stage, +/// exactly as it does for a float answer - so the histogram frames cross a boundary that is not the +/// statement's own root. +#[tokio::test] +async fn a_segmented_histogram_answer_crosses_its_collected_boundary() { + let query = "some_metric * 2 + on (__name__) some_metric * 2"; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let fused = find_fused_series(&plan) + .unwrap_or_else(|| panic!("`{query}` left the route instead of answering: {display}")); + assert_eq!( + &native_histogram_value_type().as_arrow_type(), + native_value_field(fused, &format!("`{query}`: {display}")).data_type(), + "`{query}` declared something other than the histogram it answers: {display}" + ); + let trees = fused_trees(&plan); + assert!( + trees.len() > 1, + "`{query}` answers in one node, so nothing was segmented: {trees:?}" + ); + let collected = trees + .iter() + .filter(|tree| tree.contains("Merge: canonical label-set merge")) + .collect::>(); + assert_eq!( + 2, + collected.len(), + "`{query}`: each operand pays for its own drop: {trees:?}" + ); + assert!( + fused_merges_answer_histograms(&plan), + "`{query}` collected a field that is not a histogram: {display}" + ); +} + /// A histogram fold is admitted below a pointwise operator. /// /// The gate cannot look at the root alone: `sum(h) * 2` has a binary root and the fold sits below @@ -10692,7 +11373,8 @@ async fn a_histogram_fold_is_admitted_below_a_pointwise_operator() { "avg(some_metric) + 1", "-sum(some_metric)", ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let final_node = find_final_split(&plan) @@ -10792,13 +11474,11 @@ async fn presence_and_order_aggregates_fold_histograms_natively() { #[tokio::test] async fn histogram_statements_run_natively_until_an_aggregate_folds_them() { - for query in [ - "some_metric", - "rate(some_metric[5m])", - "sum_over_time(some_metric[5m])", - "some_metric * 2", - ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + // A bare scan drops nothing, and `last_over_time` keeps the metric name, so neither hands out a + // label set two of its series could meet under. + for query in ["some_metric", "last_over_time(some_metric[5m])"] { + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; assert!( find_fused_series(&plan).is_some(), @@ -10806,6 +11486,28 @@ async fn histogram_statements_run_natively_until_an_aggregate_folds_them() { plan.display_indent_schema() ); } + // `rate`, `sum_over_time` and `some_metric * 2` all answer a histogram, and each drops the + // metadata labels over one metric - which the labels alone prove, whatever the scan's `le` + // column can hold. Nothing collects them, so no merge carries a histogram frame and the gate + // that refuses one is never reached: all three stay on the route. + for query in [ + "rate(some_metric[5m])", + "sum_over_time(some_metric[5m])", + "some_metric * 2", + ] { + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; + let display = plan.display_indent_schema().to_string(); + let fused = find_fused_series(&plan).unwrap_or_else(|| { + panic!("{query} left the native route although nothing collects it: {display}") + }); + assert!( + !matches!(fused.tree().root().expr, NativeExpr::Merge { .. }), + "{query} collects a histogram answer: {}\n{display}", + fused.tree() + ); + } // `rate` answers histograms, so an aggregate over it folds histograms - and that folds now // runs here, which is what `a_histogram_sum_or_avg_folds_natively` covers. } @@ -11565,6 +12267,7 @@ async fn mixed_native_histogram_rate_executes_real_ranges() { }, promql_annotations: Some(collector.clone()), native_fused: false, + native_binary_offer: true, }; let input = LogicalPlan::Extension(Extension { node: Arc::new( @@ -13055,6 +13758,7 @@ async fn native_scan_bounds_preserve_zero_lookback_and_overflow() { ctx: PromPlannerContext::from_eval_stmt(&build_eval_stmt("some_metric")).unwrap(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; planner.ctx.time_index_column = Some("timestamp".to_string()); planner.ctx.start = 1_000; @@ -13925,6 +14629,7 @@ async fn test_direct_or_numeric_value_types() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let left_context = direct_or_context("lhs", &["job"], "v"); let right_context = direct_or_context("rhs", &["job"], "v"); @@ -15159,6 +15864,7 @@ async fn test_mixed_or_value_aliases_do_not_replace_labels() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let left = LogicalPlanBuilder::from(scan(&left)) .project(vec![ @@ -15912,7 +16618,8 @@ async fn a_histogram_sum_or_avg_folds_natively() { "sum(rate(some_metric[5m]))", "avg by (tag_0) (some_metric)", ] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let table_provider = + build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let final_node = find_final_split(&plan) @@ -15978,11 +16685,6 @@ async fn native_composed_binary_admits_a_source_free_frame_operand() { /// shape that did not would decline to the existing plan; neither may re-offer the same expression. #[tokio::test] async fn native_composed_binary_recursion_terminates_on_a_deep_expression() { - fn count_fused(plan: &LogicalPlan) -> usize { - usize::from(is_fused_series(plan)) - + plan.inputs().into_iter().map(count_fused).sum::() - } - let depth = 32; let mut query = "hist * on(tag_0) floats".to_string(); for _ in 1..depth { @@ -15999,9 +16701,80 @@ async fn native_composed_binary_recursion_terminates_on_a_deep_expression() { assert!(!display.contains("Join:"), "{depth}-deep: {display}"); // One composition per operator, plus the leaf frames below them. assert!( - count_fused(&plan) > depth, + count_fused_nodes(&plan) > depth, "{depth}-deep: {} fused nodes: {display}", - count_fused(&plan) + count_fused_nodes(&plan) + ); +} + +/// The same match over two field kinds, `depth` binary operators deep. +fn composition_query(depth: usize) -> String { + let mut query = "hist * on(tag_0) floats".to_string(); + for _ in 1..depth { + query = format!("({query}) / on(tag_0) floats"); + } + query +} + +fn count_fused_nodes(plan: &LogicalPlan) -> usize { + usize::from(is_fused_series(plan)) + + plan + .inputs() + .into_iter() + .map(count_fused_nodes) + .sum::() +} + +/// A deep native composition plans on the stack a query worker has. +/// +/// `try_plan_native` and `try_plan_native_composed_binary` call each other once per binary operator, +/// and every level's frames are charged to the stack of the level above it. A left-deep chain of +/// matches over two field kinds 64 operators deep aborted the process - a stack overflow, not a +/// declined plan - on the two megabytes a query worker has, before `try_plan_native` grew its stack +/// for the poll of its future; 56 operators deep still planned. nextest's own test thread is not a +/// worker's, so the depths here are planned on a thread this test sizes like one, and the deepest of +/// them is twice the depth that aborted. +#[test] +fn a_deep_native_composition_plans_within_a_worker_stack() { + let depths = [10, 24, 48, 64, 128]; + let planned = std::thread::Builder::new() + .name("deep_native_composition".to_string()) + .stack_size(2 * 1024 * 1024) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a current-thread runtime"); + runtime.block_on(async { + let mut planned = Vec::new(); + for depth in depths { + let query = composition_query(depth); + let plan = build_native_fused_plan_with_provider( + &query, + true, + native_mixed_kind_provider().await, + ) + .await; + let display = plan.display_indent().to_string(); + assert!(!display.contains("Join:"), "{depth}-deep: {display}"); + // One composition per operator, plus the leaf frames below them. + assert!( + count_fused_nodes(&plan) > depth, + "{depth}-deep: {} fused nodes: {display}", + count_fused_nodes(&plan) + ); + planned.push(depth); + } + planned + }) + }) + .expect("a thread with a worker's stack") + .join() + .expect("compositions 10, 24, 48, 64 and 128 operators deep plan on a two-megabyte stack"); + assert_eq!( + depths.to_vec(), + planned, + "every depth plans as a native composition" ); } @@ -16019,6 +16792,7 @@ async fn native_composed_binary_restores_the_context_after_a_refused_operand() { ctx: PromPlannerContext::from_eval_stmt(&stmt).unwrap(), promql_annotations: None, native_fused: true, + native_binary_offer: true, }; let before = format!("{:?}", planner.ctx); let refused = planner diff --git a/src/query/src/promql/planner/test/delta.rs b/src/query/src/promql/planner/test/delta.rs index be0f5b3b28f..3913251bd8f 100644 --- a/src/query/src/promql/planner/test/delta.rs +++ b/src/query/src/promql/planner/test/delta.rs @@ -412,6 +412,7 @@ async fn delta_mixed_ranges_drop_and_float_ranges_sum() { ctx: PromPlannerContext::default(), promql_annotations: Some(collector.clone()), native_fused: false, + native_binary_offer: true, }; let left_context = direct_or_context("lhs", &["job", marker], "v"); let right_context = direct_or_context("rhs", &["job", marker], "v"); @@ -714,6 +715,7 @@ async fn binary_joins_align_only_the_temporality_marker() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let joined = planner .join_on_non_field_columns( @@ -754,6 +756,7 @@ async fn binary_joins_align_only_the_temporality_marker() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let set = planner .set_op_on_non_field_columns( @@ -810,6 +813,7 @@ async fn binary_joins_align_only_the_temporality_marker() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let set = planner .set_op_on_non_field_columns( @@ -923,6 +927,7 @@ async fn binary_joins_align_dictionary_temporality_marker_with_tagless_vector() ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, + native_binary_offer: true, }; let scan = |name, table: Arc| { LogicalPlanBuilder::scan(name, provider_as_source(table), None) diff --git a/src/query/tests/native_instant_at.rs b/src/query/tests/native_instant_at.rs index 915b06c206c..4f173120ff1 100644 --- a/src/query/tests/native_instant_at.rs +++ b/src/query/tests/native_instant_at.rs @@ -340,6 +340,27 @@ fn native_anchored_timestamp_selection_boundaries() { fn timestamp_refuses_two_series_that_collapse_to_one_label_set() { let stmt = stmt("timestamp(a @ 1)", 0, 4000, 3000); let tree = compile(&stmt, schema(1), SeriesOrdering::KeyThenTime).unwrap(); + // The refusal is the canonical pass's, so this test makes the pass the planner makes: the facts + // it hands over for a scan bound to one metric are the labels that scan's series carry - + // `__name__` among them, named by the binding - and the one metric it is bound to. `timestamp` + // drops the metadata labels, and `__type__` is one of them, so two of that scan's series can + // become one label set; the pass therefore collects the pieces above the node, and the merge is + // what reports the collision. The node itself keeps no table to check. + let source_tags = vec![vec![ + "job".to_string(), + "__type__".to_string(), + "__name__".to_string(), + ]]; + let source_metrics = vec![Some(Arc::::from("a"))]; + // The scan's key columns always decode a value here, so the labels alone decide: `__type__` in + // the tags is what keeps this drop collected. + let tree = promql::native::canonicalize::merge_label_rewrites( + &tree, + &source_tags, + &source_metrics, + true, + ) + .expect("the canonical pass accepts the tree"); let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 24)); let state = NativeExecutionState::try_new(&tree, pool, None).unwrap(); let mut executor = NativeExecutor::new(state); diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index ecb213c4ca3..fadd81ae06a 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -209,6 +209,7 @@ macro_rules! http_tests { test_remote_write_v2_histogram_flush_native_consistency, test_remote_write_v2_histogram_native_partitions, test_remote_write_v2_histogram_native_range_function, + test_remote_write_v2_histogram_native_nullable_tag_boundary, test_prometheus_remote_write_batched, test_prometheus_remote_write_batched_mixed_time_index_units, test_prometheus_remote_write_batched_interceptor_time_index_units, @@ -1701,10 +1702,13 @@ fn assert_native_plan( ) -> Value { let fused = { let mut nodes = plan_nodes_named(plan, "FusedSeriesExec"); - assert_eq!( - nodes.len(), - 1, - "{context}: expected one fused node: {nodes:?}" + // One node, or the pair a collected metadata drop is: the producer that runs the tree on + // each partition and the consumer that converts the assembled frames into rows. These + // fixture tables declare their tag column nullable, so the drop is collected and the pair is + // what a vector statement plans. + assert!( + (1..=2).contains(&nodes.len()), + "{context}: expected one fused node or the boundary's pair: {nodes:?}" ); nodes.remove(0) }; @@ -2547,6 +2551,25 @@ pub async fn test_prometheus_native_promql_route(store_type: StorageType) { instance.guard.remove_all().await; } +/// The fused node `node` is, as its own line renders it. +fn describe_node(node: &Arc) -> String { + datafusion::physical_plan::displayable(node.as_ref()) + .one_line() + .to_string() +} + +/// The fused nodes of one request, as their own node lines render them, for a failure message. +fn describe_nodes(nodes: &[Arc]) -> Vec { + nodes + .iter() + .map(|node| { + datafusion::physical_plan::displayable(node.as_ref()) + .one_line() + .to_string() + }) + .collect() +} + /// Keeps the native fused node and the raw scan the request executed, for the figures only they /// record. /// @@ -2573,12 +2596,42 @@ impl NativeFusedObserver { assert_eq!( captured.len(), 1, - "`{context}`: the request executed {} fused nodes: {captured:?}", - captured.len() + "`{context}`: the request executed {} fused nodes: {:?}", + captured.len(), + describe_nodes(&captured) ); captured.remove(0) } + /// The fused node that ran the statement's own tree. + /// + /// A statement whose metadata drop is collected runs a canonical boundary: the producer runs the + /// tree on each scan partition and the consumer above the router only converts the assembled + /// frames into the statement's rows. Every figure below is read off the producer, because it is + /// the node that binds the source, reads the scan and reports the partitions it ran on; a + /// statement with nothing collected runs that node alone. + fn take_statement(&self, context: &str) -> Arc { + let mut captured = std::mem::take(&mut *self.0.lock().unwrap()); + assert!( + !captured.is_empty(), + "`{context}`: the request executed no fused node, so it left the native route" + ); + // At most the pair a boundary is: one producer and the consumer above it. + assert!( + captured.len() <= 2, + "`{context}`: the request executed {} fused nodes: {:?}", + captured.len(), + describe_nodes(&captured) + ); + if let Some(index) = captured + .iter() + .position(|node| promql::extension_plan::is_canonical_producer_exec(node.as_ref())) + { + return captured.remove(index); + } + captured.remove(0) + } + /// The one raw scan the request just executed. /// /// The fused node reads its region through a router over a merge scan, whose region half is @@ -2772,7 +2825,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { // The body is drained, so every partition has finished and the node this statement ran is // complete; its own registrations - not the flat set the JSON view reads - carry the // per-partition figures. - let exec = observer.take(query); + let exec = observer.take_statement(query); assert_native_plan(&plan, &exec, scan_partitions, query); assert_native_partition_output(&exec, scan_partitions, expected.len(), query); @@ -2831,10 +2884,12 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { "`{query}` did not execute the fused route" ); let fused_nodes = plan_nodes_named(&plan, "FusedSeriesExec"); - assert_eq!( - fused_nodes.len(), - 1, - "`{query}`: expected one fused node: {fused_nodes:?}" + // The drop `* 100` makes over these fixture tables' nullable tag column is collected, so the + // native plan is that boundary: the producer that runs the tree and the consumer above it. An + // empty match changes which series are read, not which stages the statement is lowered into. + assert!( + (1..=2).contains(&fused_nodes.len()), + "`{query}`: expected one fused node or the boundary's pair: {fused_nodes:?}" ); let scans = plan_nodes_named(&plan, "SeriesScanExec"); assert_eq!( @@ -2845,7 +2900,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { // The body is drained, so the one fused node the request ran has finished. Its own child list is // what says how many plans it reads, and its own line is where the tree it ran is still one text: // the JSON view lists that tree's nodes as children of the operator, next to the plans it reads. - let exec = observer.take(query); + let exec = observer.take_statement(query); assert_eq!( exec.children().len(), 1, @@ -5817,9 +5872,203 @@ pub async fn test_remote_write_v2_histogram_native_range_function(store_type: St body["data"]["result"][0]["histogram"][1]["sum"], "30", "`sum_over_time` adds the sums as {PROMETHEUS_REFERENCE} does" ); + // `sum_over_time` over a native histogram drops the metadata labels, and this table's tags are + // nullable, so the canonical pass collects that drop: the answer stands on a canonical merge + // carrying histogram frames. Lowering such a boundary is two nodes, not one - a producer that + // runs the statement's own tree per partition and exports a frame, and the consumer above the + // router that converts the merged frames into the statement's rows. Both are the fused route; + // the answer asserted above is what proves the histograms survived the crossing. + let fused = observer.take_all("histogram sum_over_time"); assert_eq!( - observer.take("histogram sum_over_time").name(), - "FusedSeriesExec" + 2, + fused.len(), + "the canonical boundary is a producer and a consumer: {fused:?}" + ); + assert!( + fused.iter().all(|node| node.name() == "FusedSeriesExec"), + "the boundary executed a node that is not the fused producer: {fused:?}" + ); +} + +/// A native histogram answer crosses a canonical merge on a real metric table. +/// +/// Every tag column of a metric-engine table is nullable, so `keys_survive_metadata_drop` is false +/// on one and every statement whose kernel drops the metadata labels pays a canonical merge - the +/// `NULL`/`''` pair a drop normalizes into one label set is what the collection is for. For a +/// histogram answer that merge carries histogram frames, which is the pairing this test pins end to +/// end: the producer routes the pieces of one label set, the owner merges them with the same kernel +/// it merges floats with, and the consumer answers what it collected. +/// +/// The statement shapes here are the two sides of that crossing: `sum_over_time(h[1m])` answers the +/// histograms the merge collected, and `histogram_count(sum_over_time(h[1m]))` reduces them to +/// floats above it - so the boundary carries histograms while the statement answers a float. +pub async fn test_remote_write_v2_histogram_native_nullable_tag_boundary(store_type: StorageType) { + let observer = Arc::new(NativeFusedObserver::default()); + let plugins = Plugins::default(); + let wrapper: PhysicalPlanWrapperRef = observer.clone(); + plugins.insert(wrapper); + let instance = tests_integration::standalone::GreptimeDbStandaloneBuilder::new( + "remote_write_v2_histogram_native_nullable_tag_boundary", + ) + .with_default_store_type(store_type) + .with_query_options(QueryOptions { + experimental_native_promql: true, + ..Default::default() + }) + .with_plugin(plugins) + .build() + .await; + let server = build_test_prom_server(instance.fe_instance().clone(), false, None) + .with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap()) + .build(); + let client = TestClient::new(server.build(server.make_app()).unwrap()).await; + write_native_histogram( + &client, + &[ + (prom_store::METRIC_NAME_LABEL, "native_histogram_nullable"), + ("job", "api"), + ], + 1000, + 5, + 10.0, + ) + .await; + write_native_histogram( + &client, + &[ + (prom_store::METRIC_NAME_LABEL, "native_histogram_nullable"), + ("job", "api"), + ], + 3000, + 8, + 20.0, + ) + .await; + // The premise of this test, read off the table the writes created: the `job` tag is nullable, so + // a drop of the metadata labels is collected for the key columns rather than proved by them. + validate_data( + "native histogram nullable tag", + &client, + "select column_name, is_nullable from information_schema.columns \ + where table_schema = 'public' and table_name = 'native_histogram_nullable' \ + and column_name = 'job'", + "[[\"job\",\"YES\"]]", + ) + .await; + + // A histogram answer standing on the collected merge: `sum_over_time` over a native histogram + // drops the metadata labels, so the two writes' histograms travel through the boundary and the + // answer is the one histogram the merge collected. + observer.clear(); + let response = client + .get("/v1/prometheus/api/v1/query?query=sum_over_time(native_histogram_nullable%5B1m%5D)&time=3") + .header("x-greptime-hints", "query.experimental_native_promql=true") + .send() + .await; + assert_eq!(response.status(), StatusCode::OK); + let body: Value = response.json().await; + assert_eq!(body["data"]["resultType"], "vector", "{body}"); + assert_eq!(body["data"]["result"][0]["histogram"][0], 3.0, "{body}"); + assert_eq!( + body["data"]["result"][0]["histogram"][1]["count"], "13", + "`sum_over_time` adds the counts of the merged histograms: {body}" + ); + assert_eq!( + body["data"]["result"][0]["histogram"][1]["sum"], "30", + "`sum_over_time` adds the sums of the merged histograms: {body}" + ); + let fused = observer.take_all("histogram answer over a nullable tag"); + assert_eq!( + 2, + fused.len(), + "the collected drop is a producer and a consumer: {fused:?}" + ); + assert!( + fused.iter().all(|node| node.name() == "FusedSeriesExec"), + "the histogram boundary left the native route: {fused:?}" + ); + + // The same boundary with a float answer above it: `histogram_count` maps the histograms the + // merge collected to floats, so the frames crossing the boundary are still histograms while the + // statement publishes a float column. This is the shape the removed fallback used to refuse for + // the histogram case and the only one that ever worked here, and both now take one route. + observer.clear(); + let response = client + .get("/v1/prometheus/api/v1/query?query=histogram_count(sum_over_time(native_histogram_nullable%5B1m%5D))&time=3") + .header("x-greptime-hints", "query.experimental_native_promql=true") + .send() + .await; + assert_eq!(response.status(), StatusCode::OK); + let body: Value = response.json().await; + assert_eq!(body["data"]["resultType"], "vector", "{body}"); + assert_eq!( + body["data"]["result"][0]["value"][1].as_str(), + Some("13.0"), + "`histogram_count` counts the merged histograms' samples: {body}" + ); + let fused = observer.take_all("float answer over a histogram boundary"); + assert_eq!( + 2, + fused.len(), + "the collected drop is a producer and a consumer: {fused:?}" + ); + assert!( + fused.iter().all(|node| node.name() == "FusedSeriesExec"), + "the float answer left the native route: {fused:?}" + ); + + // `rate` is the same crossing over a window: a histogram answer whose count and sum the merge + // carries, compared against the existing plan's answer so the crossing is judged by the number + // rather than by the shape alone. + let url = "/v1/prometheus/api/v1/query?query=rate(native_histogram_nullable%5B5m%5D)&time=4"; + observer.clear(); + let native = client + .get(url) + .header("x-greptime-hints", "query.experimental_native_promql=true") + .send() + .await; + assert_eq!(native.status(), StatusCode::OK); + let native: Value = native.json().await; + let legacy = client.get(url).send().await; + assert_eq!(legacy.status(), StatusCode::OK); + let legacy: Value = legacy.json().await; + assert_eq!( + native["data"]["result"], legacy["data"]["result"], + "`rate(h[5m])`: the native answer differs from the existing plan's" + ); + // The two writes 2s apart are `count` 5 and `8` with `sum` 10 and 20, and the 5m window at the + // instant `4` extrapolates that over the range: (8 - 5) / 300s and (20 - 10) / 300s, in seconds. + assert_eq!( + ( + native["data"]["result"][0]["histogram"][1]["count"].as_str(), + native["data"]["result"][0]["histogram"][1]["sum"].as_str(), + ), + (Some("0.02"), Some("0.06666666666666667")), + "`rate(h[5m])` answered {native}" + ); + let fused = observer.take_all("histogram rate over a nullable tag"); + let descriptions = fused + .iter() + .map(|node| { + datafusion::physical_plan::displayable(node.as_ref()) + .one_line() + .to_string() + }) + .collect::>(); + assert!( + descriptions + .iter() + .all(|description| description.starts_with("FusedSeriesExec")), + "`rate(h[5m])` left the native route: {descriptions:?}" + ); + // How many fused nodes a raw window's boundary executes is a property of where its regions run + // it, so this asks what the executed plan carries instead: a canonical merge standing on the + // histogram window, which is the pairing the answer above is computed through. + assert!( + descriptions.iter().any(|description| description + .contains("Merge: canonical label-set merge") + && description.contains("Window: rate ")), + "`rate(h[5m])` executed no canonical merge over its histogram window: {descriptions:?}" ); } @@ -5897,10 +6146,20 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag (expected_count.to_string(), expected_sum.to_string()), "{query}: {body}" ); + // Scalar arithmetic over a histogram drops the metadata labels, and this table's tags are + // nullable, so the answer stands on a canonical merge that carries histogram frames. The + // boundary is two fused nodes - the producer that scales each partition's histograms and the + // consumer above the router that answers the statement - and the scaled counts and sums + // asserted above are what proves the payload crossed it intact. + let fused = observer.take_all(query); assert_eq!( - observer.take("histogram arithmetic").name(), - "FusedSeriesExec", - "{query} left the native route" + 2, + fused.len(), + "{query}: the canonical boundary is a producer and a consumer: {fused:?}" + ); + assert!( + fused.iter().all(|node| node.name() == "FusedSeriesExec"), + "{query} left the native route: {fused:?}" ); } @@ -5919,12 +6178,18 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag Some(0), "`2 / h` should drop the histogram position: {body}" ); + // `2 / h` drops the histogram position and answers nothing, but it is still the same statement + // kind on the same collected drop, so it runs through the same two-node boundary: the producer + // scales each partition's histograms, drops them, and the consumer answers the empty result. + let fused = observer.take_all("histogram arithmetic: scalar over histogram"); assert_eq!( - observer - .take("histogram arithmetic: scalar over histogram") - .name(), - "FusedSeriesExec", - "`2 / h` left the native route" + 2, + fused.len(), + "`2 / h`: the canonical boundary is a producer and a consumer: {fused:?}" + ); + assert!( + fused.iter().all(|node| node.name() == "FusedSeriesExec"), + "`2 / h` left the native route: {fused:?}" ); } @@ -7283,16 +7548,25 @@ pub async fn test_remote_write_v2_histogram_sum_and_avg_native_route(store_type: &format!("{query}: the existing path answered something else: {legacy}"), ); - // Both halves of the split executed: the partial producer folded each partition and the - // final consumer merged that state into the answer above. + // Both halves of the split executed: a partial producer that folds its own input and a final + // consumer that merges the partials above the state router. The capture holds them once per + // stage that ran the split, so the count is not pinned; what is asserted is that every node + // is a fused stage of that split and that both halves are there - one alone would mean the + // other stage fell back to the existing plan. let fused = observer.take_all(query); - assert_eq!( - fused.len(), - 2, - "{query}: expected the split's two nodes: {fused:?}" + let described = describe_nodes(&fused); + assert!( + described + .iter() + .all(|node| node.contains(" partial") || node.contains(" final")), + "{query}: a node of the capture is not one half of the split: {described:?}" ); - let consumer = answer_consumer(&fused); - assert_eq!(consumer.name(), "FusedSeriesExec"); + for half in [" partial", " final"] { + assert!( + described.iter().any(|node| node.contains(half)), + "{query}: the split ran without its{half} half: {described:?}" + ); + } } } @@ -7373,28 +7647,26 @@ pub async fn test_remote_write_v2_histogram_avg_native_partitions(store_type: St "the partitioned average is wrong: {partitioned}" ); let fused = observer.take_all("avg over partitions"); - let consumer = answer_consumer(&fused); - assert_eq!(consumer.name(), "FusedSeriesExec"); - assert_eq!( - fused.len(), - 2, - "the split's partial and final nodes both execute: {fused:?}" - ); // The producer runs once per partition that read rows, which is what makes this a merge and not - // a single fold. + // a single fold, so the figures this test reads are the partial stage's: it is the node whose + // tree carries the fold before the state router, and it is the only node that reports them. let producer = fused .iter() - .find(|node| !Arc::ptr_eq(node, consumer)) - .expect("the partial producer"); - let metrics = producer.metrics().expect("producer metrics"); + .find(|node| describe_node(node).contains(" partial")) + .expect("the split's partial producer"); + // The per-partition figures are the ones the node registered per partition, which is the view + // `node_metric_set` reports; the node's own aggregated set sums them into one entry. + let metrics = node_metric_set(producer); let mut partitions: BTreeMap = BTreeMap::new(); for metric in metrics.iter() { - if metric.value().name() == "native.root.output_values" - && let MetricValue::Count { count, .. } = metric.value() - { - *partitions - .entry(metric.partition().expect("partition label")) - .or_insert(0usize) += count.value(); + let Some(partition) = metric.partition() else { + continue; + }; + let MetricValue::Count { count, .. } = metric.value() else { + continue; + }; + if metric.value().name() == "native.root.output_values" { + *partitions.entry(partition).or_insert(0usize) += count.value(); } } assert_eq!(