From 3f8d718a3015f3c55720489d739de0ef036d7ddd Mon Sep 17 00:00:00 2001 From: discord9 Date: Wed, 18 Nov 2026 11:09:12 +0800 Subject: [PATCH] revert(promql): revert #10412, merged without approval (#10495) Revert "perf(promql): stop building label-set tables where they cannot collide (#10412)" This reverts commit 1fd9069418aa7a69ab59a71e212d03ab4b0bbc19. --- Cargo.lock | 1 - src/datatypes/src/series/time.rs | 76 +- src/datatypes/src/series/time/tests.rs | 75 +- src/mito2/src/series_index/maintenance.rs | 161 +- src/promql/src/aggregate.rs | 378 ++-- src/promql/src/aggregate/state.rs | 258 +-- src/promql/src/aggregate/tests.rs | 207 +- src/promql/src/extension_plan.rs | 4 +- src/promql/src/extension_plan/fused_series.rs | 84 +- .../src/extension_plan/fused_series/tests.rs | 77 +- .../src/extension_plan/series_repartition.rs | 18 +- .../series_repartition/tests.rs | 179 +- src/promql/src/grid_series.rs | 52 +- src/promql/src/grid_series/tests.rs | 247 +-- src/promql/src/histogram_eval/aggregate.rs | 24 +- .../src/histogram_eval/float_histogram.rs | 62 +- src/promql/src/histogram_eval/tests.rs | 86 +- src/promql/src/instant_eval.rs | 24 +- src/promql/src/instant_eval/tests.rs | 41 +- src/promql/src/native/canonicalize.rs | 196 +- src/promql/src/native/canonicalize/tests.rs | 813 +------- src/promql/src/native/execute.rs | 548 +++-- src/promql/src/native/execute/tests.rs | 1811 ++++++----------- src/promql/src/native/mod.rs | 1 + src/promql/src/native/plan.rs | 558 +---- src/promql/src/native/plan/tests.rs | 354 +--- src/promql/src/native/runs.rs | 143 ++ src/promql/src/native/segment.rs | 15 + src/promql/src/range_eval.rs | 177 +- src/promql/src/range_eval/function.rs | 231 +-- src/promql/src/range_eval/histogram.rs | 5 +- src/promql/src/range_eval/tests.rs | 232 --- src/promql/src/select.rs | 113 +- src/promql/src/select/tests.rs | 146 +- src/promql/src/vector_match.rs | 149 +- src/promql/src/vector_match/tests.rs | 199 +- src/query/Cargo.toml | 1 - src/query/src/metrics.rs | 30 - src/query/src/promql/planner.rs | 118 +- src/query/src/promql/planner/native.rs | 412 +--- src/query/src/promql/planner/test.rs | 932 +-------- src/query/src/promql/planner/test/delta.rs | 5 - src/query/tests/native_instant_at.rs | 21 - tests-integration/tests/http.rs | 366 +--- 44 files changed, 1944 insertions(+), 7686 deletions(-) create mode 100644 src/promql/src/native/runs.rs diff --git a/Cargo.lock b/Cargo.lock index 6c3fef5dc14..fd31d88c4dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12087,7 +12087,6 @@ dependencies = [ "snafu 0.8.6", "sql", "sqlparser", - "stacker", "store-api", "substrait 1.4.0-alpha.0", "table", diff --git a/src/datatypes/src/series/time.rs b/src/datatypes/src/series/time.rs index bf5b50adf24..eae023a1fc3 100644 --- a/src/datatypes/src/series/time.rs +++ b/src/datatypes/src/series/time.rs @@ -148,41 +148,19 @@ impl EvalGrid { self.timestamps.get_or_try_init(|| { let len = usize::try_from(self.spec.len) .map_err(|_| invalid_series("grid length overflows usize"))?; - Ok(self.gather_timestamps(0, len)?.into()) + let mut values = Vec::new(); + values + .try_reserve_exact(len) + .map_err(|_| invalid_series("grid timestamp allocation failed"))?; + for index in 0..len { + let timestamp = + (self.spec.start_ms as i128) + (index as i128) * (self.spec.step_ms as i128); + values.push(timestamp as i64); + } + Ok(values.into()) }) } - /// Whether the whole timestamp array exists already. - /// - /// A consumer that pays for the array -- an execution reserving what it is about to allocate -- - /// asks this before it materializes a grid: a grid that is already materialized was charged to - /// whoever materialized it, and a second consumer must not charge it again. - pub fn is_materialized(&self) -> bool { - self.timestamps.get().is_some() - } - - /// Returns the timestamps of the positions `start..start + len` as an array of their own, - /// leaving the grid unmaterialized. - /// - /// Every value is [`Self::timestamp_at`]'s, in position order, so gathering a window of the - /// grid costs its own length and not the length of the grid it is cut from. - fn gather_timestamps(&self, start: usize, len: usize) -> Result> { - let mut values = Vec::new(); - values - .try_reserve_exact(len) - .map_err(|_| invalid_series("grid timestamp allocation failed"))?; - let end = start.checked_add(len).context(InvalidSeriesSnafu { - reason: "grid range overflow".to_string(), - })?; - for position in start..end { - let Some(timestamp) = self.timestamp_at(position) else { - return invalid("grid range exceeds its grid"); - }; - values.push(timestamp); - } - Ok(values) - } - fn timestamp_at(&self, position: usize) -> Option { if position as u64 >= self.spec.len { return None; @@ -233,22 +211,9 @@ impl GridRange { pub fn is_empty(&self) -> bool { self.len == 0 } - /// Returns the timestamps of the positions this range names. - /// - /// A range covering the whole grid shares the grid's array, which that grid has to materialize - /// for any reader of all of it. A range cut out of a larger grid computes the `len` timestamps - /// it names instead, each one [`EvalGrid::timestamp_at`]'s for its position: reading a window - /// out of a large grid costs the window's rows, and leaves a deferred grid deferred. + /// Returns this range as a zero-copy Arrow buffer slice. pub fn timestamps(&self) -> Result> { - if self.covers_grid() { - return Ok(self.grid.timestamps()?.clone()); - } - Ok(self.grid.gather_timestamps(self.start, self.len)?.into()) - } - - /// Whether this range names every position of its grid. - fn covers_grid(&self) -> bool { - self.start == 0 && u64::try_from(self.len).is_ok_and(|len| len == self.grid.spec.len) + Ok(self.grid.timestamps()?.slice(self.start, self.len)) } } @@ -844,20 +809,13 @@ impl TimeAxis { /// Returns the timestamps of the carried rows, in compact-row order. /// - /// An axis covering the whole grid lends the grid's array; a grid axis holding fewer positions - /// than its grid computes the rows it carries and lends nothing, so gathering a window out of - /// a large grid costs the window and leaves a deferred grid deferred. An axis of several runs, - /// and a chunked explicit axis, allocate one array of the carried rows. + /// A single-run grid axis lends its buffer without copying; an axis of several runs, and a + /// chunked explicit axis, allocate one array of the carried rows. pub fn timestamps(&self) -> Result> { match self { - Self::Grid(range) => { - if !range.covers_grid() { - return Ok(Cow::Owned( - range.grid.gather_timestamps(range.start(), range.len())?, - )); - } - Ok(Cow::Borrowed(&range.grid.timestamps()?[..])) - } + Self::Grid(range) => Ok(Cow::Borrowed( + &range.grid.timestamps()?[range.start()..range.start() + range.len()], + )), Self::GridRanges(ranges) => { let mut timestamps = Vec::with_capacity(ranges.len()); for (position, _) in self.positions() { diff --git a/src/datatypes/src/series/time/tests.rs b/src/datatypes/src/series/time/tests.rs index 33d5321a08d..73e165f6f5f 100644 --- a/src/datatypes/src/series/time/tests.rs +++ b/src/datatypes/src/series/time/tests.rs @@ -118,67 +118,11 @@ fn deferred_grid_allocation_failure_is_reported_and_not_cached() { Arc::new(EvalGrid::try_deferred(GridSpec::try_new(i64::MIN, 1, len).unwrap()).unwrap()); assert!(grid.timestamps().is_err()); assert!(grid.timestamps.get().is_none()); - // A range naming every point of that grid asks for the array that does not fit, and the - // failure is still the series error rather than an abort. - let whole = GridRange::try_new(grid.clone(), 0, len as usize).unwrap(); - assert!(whole.timestamps().is_err()); - assert!(TimeAxis::Grid(whole).timestamps().is_err()); + let range = GridRange::try_new(grid.clone(), 0, 1).unwrap(); + assert!(range.timestamps().is_err()); + assert!(TimeAxis::Grid(range).timestamps().is_err()); assert!(grid.timestamps().is_err()); assert!(grid.timestamps.get().is_none()); - // A range that is nearly the whole grid gathers its own rows, and its allocation fails too: - // the gathered array is the range's, so the failure is reported rather than aborted. - let nearly = GridRange::try_new(grid.clone(), 0, len as usize - 1).unwrap(); - assert!(nearly.timestamps().is_err()); - assert!(TimeAxis::Grid(nearly).timestamps().is_err()); - assert!(grid.timestamps.get().is_none()); -} - -/// A window cut out of a large grid computes the rows it names instead of materializing the grid. -/// -/// The gathered values are the grid's own points, so a subrange reads the same timestamps it did -/// when it was a slice of the whole array; what changes is the cost, which is the window's rows -/// and no longer the grid's. -#[test] -fn subrange_timestamps_gather_only_the_rows_they_name() { - let len = (isize::MAX as u64 / std::mem::size_of::() as u64) + 1; - let grid = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(-5, 2, len).unwrap()).unwrap()); - let range = GridRange::try_new(grid.clone(), 7, 3).unwrap(); - assert!(!grid.is_materialized()); - let gathered = range.timestamps().unwrap(); - assert_eq!(&gathered[..], &[9, 11, 13]); - let axis = TimeAxis::Grid(range); - let axis_timestamps = axis.timestamps().unwrap(); - assert_eq!(axis_timestamps.as_ref(), &[9, 11, 13]); - // Every gathered point is what the axis answers for the row holding it. - for (row, timestamp) in axis_timestamps.iter().enumerate() { - assert_eq!(*timestamp, axis.timestamp(row).unwrap()); - } - assert!( - !grid.is_materialized(), - "the subrange materialized its grid" - ); - - // A range naming the whole grid still shares the grid's one array. - let small = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(10, 10, 4).unwrap()).unwrap()); - let all = GridRange::try_new(small.clone(), 0, 4).unwrap(); - assert!(!small.is_materialized()); - let shared = all.timestamps().unwrap(); - assert!( - small.is_materialized(), - "the whole-grid range did not materialize the grid it shares" - ); - assert_eq!(shared.as_ptr(), small.timestamps().unwrap().as_ptr()); - assert_eq!( - TimeAxis::Grid(all).timestamps().unwrap().as_ref(), - &[10, 20, 30, 40] - ); - - // An empty range at the end of a grid has no row to gather and materializes nothing. - let grid = Arc::new(EvalGrid::try_deferred(GridSpec::try_new(0, 1, 3).unwrap()).unwrap()); - let empty = GridRange::try_new(grid.clone(), 3, 0).unwrap(); - assert!(empty.timestamps().unwrap().is_empty()); - assert!(TimeAxis::Grid(empty).timestamps().unwrap().is_empty()); - assert!(!grid.is_materialized()); } #[test] @@ -223,20 +167,11 @@ fn grid_materialization_revalidates_public_specs_and_shares_ranges_by_value() { let b = GridRange::try_new(Arc::new(EvalGrid::try_new(grid.spec()).unwrap()), 1, 3).unwrap(); assert_eq!(a, b); let values = a.timestamps().unwrap(); - // A subrange computes its own rows rather than slicing the grid's array, which it no longer - // shares: what it answers is still the grid's points, one per position it names. - assert_ne!( + assert_eq!( values.as_ptr(), - grid.timestamps().unwrap().as_ptr().wrapping_add(1), - "the subrange borrowed the grid's array" + grid.timestamps().unwrap().as_ptr().wrapping_add(1) ); assert_eq!(&values[..], &[-2, 1, 4]); - let whole = GridRange::try_new(grid.clone(), 0, 5).unwrap(); - assert_eq!( - whole.timestamps().unwrap().as_ptr(), - grid.timestamps().unwrap().as_ptr(), - "a whole-grid range copied the array it could share" - ); assert!(GridRange::try_new(grid.clone(), 5, 0).unwrap().is_empty()); assert!(GridRange::try_new(grid.clone(), 4, 2).is_err()); assert!(GridRange::try_new(grid, usize::MAX, 1).is_err()); diff --git a/src/mito2/src/series_index/maintenance.rs b/src/mito2/src/series_index/maintenance.rs index d77e9551e59..c127fd3a42d 100644 --- a/src/mito2/src/series_index/maintenance.rs +++ b/src/mito2/src/series_index/maintenance.rs @@ -226,52 +226,6 @@ pub(crate) async fn reconcile_series_indexes_with_mode( Ok(stats) } -/// How many sources one samples build reads at once. The writer bounds blocks and output sizes on -/// its own, so this only bounds the memory one pinned source group holds. -const SAMPLES_SOURCES_PER_BUILD: usize = 64; - -/// Splits uncovered sources into samples build groups, by UTC-day window and then by SST level. -/// -/// A samples artifact records the files it was built from and the days each of them contributed, so -/// a group is an identity, not a batch: every output of one group names the same sources, and -/// coverage is read back per group. Cutting the sources by file id alone - the previous behaviour - -/// put every level and every window into the same arbitrary chunks, so one new file re-cut them and -/// the days of unrelated levels were rebuilt. Grouping by window and level keeps a group's boundary -/// where the data puts it: a file of another level or another window joins its own group, and the -/// outputs already published for a group stay reusable when a source is added beside it. -/// -/// Windows sort before levels so the days of one window are built next to each other, and the files -/// inside a group are ordered by id so a build reads them in the same order every time. -fn sample_source_groups(files: &[FileHandle]) -> Vec> { - use std::collections::BTreeMap; - - use crate::series_index::builder::sample_day; - - let mut groups = BTreeMap::<(i64, i64, u8), Vec>::new(); - for file in files { - // The span the file covers is the window it writes into. A file holding several days is a - // window of its own here and still reports the days it actually filled. - let (start, end) = file.time_range(); - let (start, end) = (sample_day(start), sample_day(end)); - groups - .entry((start.min(end), start.max(end), file.level())) - .or_default() - .push(file.clone()); - } - groups - .into_values() - .map(|mut group| { - group.sort_unstable_by(|left, right| { - left.file_id() - .file_id() - .as_bytes() - .cmp(right.file_id().file_id().as_bytes()) - }); - group - }) - .collect() -} - /// Builds a changed snapshot, retaining reusable indexes and removing obsolete coverage. /// Returns `None` when no range or series indexes were added or removed. #[allow(clippy::too_many_arguments)] @@ -504,27 +458,26 @@ async fn build_index_version( .values() .flat_map(|handle| handle.entry().source_file_ids.iter().copied()) .collect(); - let uncovered: Vec<_> = files + let mut uncovered: Vec<_> = files .iter() .filter(|file| !covered.contains(&file.file_id().file_id())) .cloned() .collect(); - for sources in sample_source_groups(&uncovered) { - for sources in sources.chunks(SAMPLES_SOURCES_PER_BUILD) { - let handles = crate::series_index::builder::build_sample_windows( - store, - region, - metadata, - sources, - backend.as_ref(), - purger, - ) - .await?; - for handle in handles { - unpublished.1.push(handle.clone()); - stats.built_samples += 1; - daily.insert(handle.entry().file_id, handle); - } + uncovered.sort_unstable_by_key(|file| file.file_id().file_id().to_string()); + for sources in uncovered.chunks(64) { + let handles = crate::series_index::builder::build_sample_windows( + store, + region, + metadata, + sources, + backend.as_ref(), + purger, + ) + .await?; + for handle in handles { + unpublished.1.push(handle.clone()); + stats.built_samples += 1; + daily.insert(handle.entry().file_id, handle); } } stats.removed_samples += samples.keys().filter(|id| !daily.contains_key(id)).count(); @@ -740,95 +693,13 @@ fn publish_index_version(region: &MitoRegionRef, next: Arc) #[cfg(test)] mod tests { use std::collections::HashMap; - use std::sync::Arc; use common_time::Timestamp; use object_store::services::Memory; use super::*; - use crate::series_index::builder::sample_day; use crate::series_index::catalog::{RangeIndexEntry, SeriesIndexEntry}; use crate::series_index::purger::series_index_channel; - use crate::sst::file::{FileHandle, FileMeta}; - use crate::sst::file_purger::NoopFilePurger; - - /// A source file covering whole UTC days `start_day..=end_day` at `level`. - fn samples_source(id: u8, level: u8, start_day: i64, end_day: i64) -> FileHandle { - const DAY_MS: i64 = 86_400_000; - FileHandle::new( - FileMeta { - file_id: FileId::from_bytes([id; 16]), - time_range: ( - Timestamp::new_millisecond(start_day * DAY_MS), - Timestamp::new_millisecond(end_day * DAY_MS + DAY_MS - 1), - ), - level, - ..Default::default() - }, - Arc::new(NoopFilePurger), - ) - } - - /// Sources are grouped by the window they cover and by their level, not cut by file id. - /// - /// Every group is one window of one level, so a file that joins a level later cannot move the - /// boundary of a group built for another level, and the group's days are the days it covers. - #[test] - fn samples_sources_are_grouped_by_window_and_level() { - let mut files = vec![ - samples_source(1, 0, 0, 0), - samples_source(2, 0, 0, 0), - samples_source(3, 1, 0, 0), - samples_source(4, 0, 5, 5), - samples_source(5, 1, 5, 7), - samples_source(6, 0, 5, 5), - ]; - // Grouping must not depend on the ids or on the order the files arrive in. - files.rotate_left(2); - - let groups = sample_source_groups(&files); - let describe = |group: &[FileHandle]| { - let days: Vec<(i64, i64)> = group - .iter() - .map(|file| { - let (start, end) = file.time_range(); - (sample_day(start), sample_day(end)) - }) - .collect(); - (group[0].level(), days) - }; - assert_eq!( - vec![ - (0, vec![(0, 0), (0, 0)]), - (1, vec![(0, 0)]), - (0, vec![(5, 5), (5, 5)]), - (1, vec![(5, 7)]), - ], - groups - .iter() - .map(|group| describe(group)) - .collect::>(), - "one window of one level per group, ordered by window then level" - ); - // Every source is placed exactly once, and no group mixes a level or a window. - let mut grouped: Vec<_> = groups - .iter() - .flatten() - .map(|file| file.file_id().file_id().as_bytes().to_vec()) - .collect(); - let mut sources: Vec<_> = files - .iter() - .map(|file| file.file_id().file_id().as_bytes().to_vec()) - .collect(); - grouped.sort_unstable(); - sources.sort_unstable(); - assert_eq!(sources, grouped); - for group in &groups { - let (level, days) = describe(group); - assert!(group.iter().all(|file| file.level() == level)); - assert!(days.windows(2).all(|pair| pair[0] == pair[1])); - } - } #[rstest::rstest] fn test_prune_index_version( diff --git a/src/promql/src/aggregate.rs b/src/promql/src/aggregate.rs index af4d8ad49d9..59ab98912aa 100644 --- a/src/promql/src/aggregate.rs +++ b/src/promql/src/aggregate.rs @@ -123,24 +123,6 @@ pub(crate) enum HistogramSamples { Folded, } -impl AggregateOp { - /// The words of `f64` one field's state of this aggregation keeps for one grid point. - /// - /// The last word is the presence word: the op's own count where its state carries one, so a - /// zero count reads as unseen, and a `0`/`1` flag where it does not. The words before it are - /// the components the op folds, in [`Self::state_fields`] order. A state therefore costs what - /// its own operation needs instead of what the widest operation needs: a block holds - /// `BLOCK_POINTS * fields * state_words * size_of::()` bytes however wide the op is. - pub(crate) const fn state_words(self) -> usize { - match self { - Self::Count | Self::Group => 1, - Self::Min | Self::Max => 2, - Self::Sum | Self::Stddev | Self::Stdvar => 3, - Self::Avg => 5, - } - } -} - /// What `op` does with a native histogram sample. pub(crate) fn histogram_samples(op: AggregateOp) -> HistogramSamples { match op { @@ -236,45 +218,16 @@ impl FloatAggregator { ) -> Result<()> { validate_aggregate_series(&self.schema, series, self.op)?; let existing_bytes = self.state_bytes(); - let group_capacity = self.groups.capacity(); - let map_capacity = self.group_index.capacity(); - // One admission for everything the fold can allocate before it writes a sample: the copy of - // this series' metric the grouping makes, the group's own copy of those labels and its - // key, the labels vector, and the group vector and index that may grow. What only a new - // group pays for is admitted whether or not this series opens one, so a chunk costs one - // pool call here instead of one per step, and a refusal still arrives before any of it is - // allocated. - // - // The grouping copy, the second workspace the copy is grouped into, and the group's - // retained labels and key are each a copy or a subset of this series' labels, so three - // copies of the one workspace bound all of them. - let label_workspace = grouping_workspace_bytes(&series.metric); - budget( - existing_bytes - .saturating_add(label_workspace.saturating_mul(3)) - .saturating_add(pending_bytes(self.schema.fields.len())) - .saturating_add( - self.labels - .len() - .saturating_mul(std::mem::size_of::<&str>()), - ) - // A realloc that cannot grow in place keeps both buffers alive while it copies, - // so a growing step admits the new buffer beside the old one, which the retained - // state already holds. The map is admitted the same way. - .saturating_add(if self.groups.len() == group_capacity { - group_capacity - .saturating_mul(2) - .max(4) - .saturating_mul(size_of::()) - } else { - 0 - }) - .saturating_add(if self.group_index.len() == map_capacity { - hash_map_backing_bytes(next_hash_capacity(map_capacity)) - } else { - 0 - }), - )?; + let grouping_workspace = grouping_workspace_bytes(&series.metric); + let mut workspace_bytes = + grouping_workspace.saturating_add(pending_bytes(self.schema.fields.len())); + budget(existing_bytes.saturating_add(workspace_bytes))?; + workspace_bytes = workspace_bytes.saturating_add( + self.labels + .len() + .saturating_mul(std::mem::size_of::<&str>()), + ); + budget(existing_bytes.saturating_add(workspace_bytes))?; let labels: Vec<&str> = self.labels.iter().map(|label| &**label).collect(); let grouping = if self.by { Grouping::By(&labels) @@ -282,12 +235,51 @@ impl FloatAggregator { Grouping::Without(&labels) }; let metric = grouping.apply(&series.metric)?; + let key_bytes = metric + .tags() + .len() + .saturating_add(1) + .saturating_mul(std::mem::size_of::<(Arc, Arc)>()); + workspace_bytes = workspace_bytes + .saturating_add(grouping_workspace_bytes(&metric)) + .saturating_add(key_bytes); + budget(existing_bytes.saturating_add(workspace_bytes))?; let key = label_key(&metric); let key_retained = key .capacity() .saturating_mul(size_of::<(Arc, Arc)>()); let new_group = !self.group_index.contains_key(&key); + let group_capacity = self.groups.capacity(); + let map_capacity = self.group_index.capacity(); let old_map_backing = hash_map_backing_bytes(map_capacity); + if new_group { + let next_group_capacity = if self.groups.len() == group_capacity { + group_capacity.saturating_mul(2).max(4) + } else { + group_capacity + }; + let group_realloc_bytes = if next_group_capacity > group_capacity { + next_group_capacity.saturating_mul(size_of::()) + } else { + 0 + }; + budget( + existing_bytes + .saturating_add(metric_bytes(&metric)) + .saturating_add(key_bytes) + // A realloc that cannot grow in place keeps both buffers alive while it + // copies, so a growing step admits the old buffer beside the new one -- + // the same reason the map below admits both tables. The old buffer alone + // is already in `existing_bytes`; what is added here is the new one. + .saturating_add(group_realloc_bytes) + .saturating_add(if self.group_index.len() == map_capacity { + hash_map_backing_bytes(next_hash_capacity(map_capacity)) + } else { + 0 + }) + .saturating_add(workspace_bytes), + )?; + } let group_capacity_growth = if new_group && self.groups.len() == group_capacity { group_capacity.saturating_mul(2).max(4) } else { @@ -383,8 +375,7 @@ impl FloatAggregator { .len() .saturating_mul(BLOCK_POINTS) .saturating_mul(self.schema.fields.len()) - .saturating_mul(self.op.state_words()) - .saturating_mul(std::mem::size_of::()), + .saturating_mul(std::mem::size_of::()), ) .saturating_add(btree_backing_bytes(group.blocks.len())) .saturating_add(histogram_backing_bytes(group.histograms.len())) @@ -468,8 +459,6 @@ fn finish_groups_budgeted( DataFusionError::Execution(format!("cannot reserve aggregate row: {error}")) })?; let mut completed_output_bytes = 0usize; - let words = op.state_words(); - let point_words = width.saturating_mul(words); for group in groups { // A group that folded histogram samples answers histogram columns; one that folded floats // keeps the float path below, so a group without a single histogram payload is built @@ -498,7 +487,7 @@ fn finish_groups_budgeted( } // One builder serves every group, so each result series starts explicitly. builder.begin_series(); - let (rows, runs) = group_rows_runs(&group, width, op); + let (rows, runs) = group_rows_runs(&group, width); let finish_bytes = finish_working_bytes(rows, runs, width); budget( state @@ -510,16 +499,12 @@ fn finish_groups_budgeted( builder.reserve_rows(rows)?; for (&block, states) in &group.blocks { // The last block can extend past the grid; only points with a sample are pushed. - for (offset, point) in states.chunks_exact(point_words).enumerate() { - if !point.chunks_exact(words).any(|state| state_seen(state, op)) { + for (offset, point) in states.chunks_exact(width).enumerate() { + if !point.iter().any(|state| state.seen) { continue; } row.clear(); - row.extend( - point - .chunks_exact(words) - .map(|state| state_finish(state, op)), - ); + row.extend(point.iter().map(|state| state.finish(op))); builder.push(block * BLOCK_POINTS + offset, &row)?; } } @@ -556,12 +541,11 @@ fn finish_histogram_group( // A field's column is one type. The fold already refuses a histogram where it had folded a // float at that position; a float folded after a histogram is caught here, where the column // type is chosen for the whole group. - let words = op.state_words(); - if group.blocks.values().any(|states| { - states - .chunks_exact(words) - .any(|state| state_seen(state, op)) - }) { + if group + .blocks + .values() + .any(|states| states.iter().any(|state| state.seen)) + { return Err(DataFusionError::Execution( "a native aggregate cannot mix float and histogram samples in one field".into(), )); @@ -713,18 +697,16 @@ fn finish_groups( finish_groups_budgeted(schema, op, groups, 0, |_| Ok(())) } -fn group_rows_runs(group: &Group, width: usize, op: AggregateOp) -> (usize, usize) { +fn group_rows_runs(group: &Group, width: usize) -> (usize, usize) { if width == 0 { return (0, 0); } - let words = op.state_words(); - let point_words = width * words; let mut rows = 0usize; let mut runs = 0usize; let mut previous: Option = None; for (&block, states) in &group.blocks { - for (offset, point) in states.chunks_exact(point_words).enumerate() { - if point.chunks_exact(words).any(|state| state_seen(state, op)) { + for (offset, point) in states.chunks_exact(width).enumerate() { + if point.iter().any(|state| state.seen) { let index = block.saturating_mul(BLOCK_POINTS).saturating_add(offset); rows = rows.saturating_add(1); if previous.is_none_or(|previous| previous.checked_add(1) != Some(index)) { @@ -1111,7 +1093,7 @@ fn next_hash_capacity(capacity: usize) -> usize { /// words; the leaf form is smaller than that. The header allowance is generous on purpose, so the /// bound does not depend on how a toolchain lays the node out. fn btree_node_bytes() -> usize { - 11 * size_of::<(usize, Box<[f64]>)>() + 16 * size_of::() + 11 * size_of::<(usize, Box<[FloatState]>)>() + 16 * size_of::() } /// An upper bound on the bytes a B-tree of `entries` holds. @@ -1166,21 +1148,11 @@ fn label_key(metric: &SeriesLabels) -> LabelKey { /// input never expands to the full grid and dense input costs one allocation per block. const BLOCK_POINTS: usize = 64; -/// Bytes one allocated block keeps: a state per point per field, [`AggregateOp::state_words`] -/// words of `f64` each. The fold, the merge and the recount all size a block through this, so what -/// is admitted and what is allocated cannot drift apart. -fn block_bytes(width: usize, op: AggregateOp) -> usize { - BLOCK_POINTS - .saturating_mul(width) - .saturating_mul(op.state_words()) - .saturating_mul(size_of::()) -} - struct Group { metric: SeriesLabels, /// Keyed by absolute grid index / `BLOCK_POINTS`; each block holds one state per field for - /// each of its points, point by point, each state as many `f64` words as the op folds. - blocks: BTreeMap>, + /// each of its points, point by point. + blocks: BTreeMap>, /// The histogram states of this group, keyed by absolute grid position and field. /// /// Only an aggregate that folds a payload allocates one, and only where it saw a histogram @@ -1255,20 +1227,6 @@ struct FieldFold { op: AggregateOp, } -impl FieldFold { - /// Words one point of this fold's block keeps: every field's state, side by side. - fn point_words(self) -> usize { - self.width.saturating_mul(self.op.state_words()) - } - - /// Where this field's state of the point at `offset` starts inside a block. - fn state_start(self, offset: usize) -> usize { - offset - .saturating_mul(self.point_words()) - .saturating_add(self.field_index.saturating_mul(self.op.state_words())) - } -} - impl Group { /// Folds one series' float samples into this group's blocks, one field at a time. /// @@ -1353,15 +1311,14 @@ impl Group { let block = index / BLOCK_POINTS; let (states, admitted) = self.block_states(block, fold, budget, retained)?; retained = admitted; - let words = fold.op.state_words(); - let start = fold.state_start(index % BLOCK_POINTS); - FloatState::new(&mut states[start..start + words], fold.op).update(sample); + states[(index % BLOCK_POINTS) * fold.width + fold.field_index] + .update(fold.op, sample); while let Some(&(at, sample)) = samples.peek() && at / BLOCK_POINTS == block { samples.next(); - let start = fold.state_start(at % BLOCK_POINTS); - FloatState::new(&mut states[start..start + words], fold.op).update(sample); + states[(at % BLOCK_POINTS) * fold.width + fold.field_index] + .update(fold.op, sample); } } } @@ -1379,11 +1336,13 @@ impl Group { fold: FieldFold, budget: &mut impl FnMut(usize) -> Result<()>, retained: usize, - ) -> Result<(&'a mut [f64], usize)> { + ) -> Result<(&'a mut [FloatState], usize)> { let new_block = !self.blocks.contains_key(&block); - let (states_bytes, backing_delta) = if new_block { + let (block_bytes, backing_delta) = if new_block { ( - block_bytes(fold.width, fold.op), + BLOCK_POINTS + .saturating_mul(fold.width) + .saturating_mul(size_of::()), btree_backing_bytes(self.blocks.len() + 1) .saturating_sub(btree_backing_bytes(self.blocks.len())), ) @@ -1392,14 +1351,13 @@ impl Group { }; let retained = retained .saturating_add(backing_delta) - .saturating_add(states_bytes); + .saturating_add(block_bytes); if new_block { budget(retained)?; } - let states = self - .blocks - .entry(block) - .or_insert_with(|| vec![0.0; BLOCK_POINTS * fold.point_words()].into_boxed_slice()); + let states = self.blocks.entry(block).or_insert_with(|| { + vec![FloatState::default(); BLOCK_POINTS * fold.width].into_boxed_slice() + }); Ok((&mut states[..], retained)) } } @@ -1429,10 +1387,8 @@ impl Group { // A field's column is either float or histogram, so a position that already folded a float // cannot also fold a histogram. The other order is caught when the fold is finished, which // is where the column type is chosen. - let words = fold.op.state_words(); - let start = fold.state_start(position % BLOCK_POINTS); if let Some(states) = self.blocks.get(&(position / BLOCK_POINTS)) - && state_seen(&states[start..start + words], fold.op) + && states[(position % BLOCK_POINTS) * fold.width + fold.field_index].seen { return Err(DataFusionError::Execution( "a native aggregate cannot mix float and histogram samples in one field".into(), @@ -1863,137 +1819,95 @@ fn histogram_backing_bytes(entries: usize) -> usize { entries.div_ceil(5).saturating_mul(histogram_node_bytes()) } -/// The float part of Prometheus' `groupedAggregation`, as a view of one state's words. -/// -/// A state is [`AggregateOp::state_words`] words of the block it sits in: the components the op -/// folds, then the presence word. The view reads and writes that layout in place, so folding -/// allocates nothing per state and an op that keeps less memory keeps less. -struct FloatState<'a> { - words: &'a mut [f64], - op: AggregateOp, +/// The float part of Prometheus' `groupedAggregation`. +#[derive(Clone, Copy, Default)] +struct FloatState { + seen: bool, + value: f64, + mean: f64, + compensation: f64, + count: f64, + incremental_mean: bool, } -/// Whether the state in `words` has seen a sample: its presence word is set. -/// -/// The presence word is the last one, which is the op's count where its state carries one - a -/// state that has folded nothing has no count - and a `0`/`1` flag where it does not. -fn state_seen(words: &[f64], op: AggregateOp) -> bool { - words[op.state_words() - 1] != 0.0 -} - -/// The value the state in `words` finishes to, or `None` if it has seen no sample. -fn state_finish(words: &[f64], op: AggregateOp) -> Option { - if !state_seen(words, op) { - return None; - } - Some(match op { - // `value`, its compensation, then the presence flag. - AggregateOp::Sum => words[0] + words[1], - // `value`, `compensation`, `mean`, the incremental flag, then the count. - AggregateOp::Avg if words[3] != 0.0 => words[2] + words[1], - AggregateOp::Avg => words[0] / words[4] + words[1] / words[4], - AggregateOp::Count => words[0], - AggregateOp::Min | AggregateOp::Max => words[0], - AggregateOp::Group => 1.0, - // `value`, `mean`, then the count. - AggregateOp::Stdvar => words[0] / words[2], - AggregateOp::Stddev => (words[0] / words[2]).sqrt(), - }) -} - -impl<'a> FloatState<'a> { - /// The state at `words`, which is `op`'s layout: [`AggregateOp::state_words`] words. - fn new(words: &'a mut [f64], op: AggregateOp) -> Self { - debug_assert_eq!(words.len(), op.state_words()); - Self { words, op } - } - - /// Folds one sample in, initializing the state if it had seen none. - fn update(self, sample: f64) { - let (op, words) = (self.op, self.words); - if !state_seen(words, op) { - match op { - // The count is the presence word. - AggregateOp::Count => words[0] = 1.0, - // Presence alone: `group` answers 1 for any kind of sample, NaN included, so the - // sample itself is not kept - a zero sample would read as unseen. - AggregateOp::Group => words[0] = 1.0, - // `value`, then the presence flag. - AggregateOp::Min | AggregateOp::Max => { - words[0] = sample; - words[1] = 1.0; - } - // `value`, its compensation, then the presence flag. - AggregateOp::Sum => { - words[0] = sample; - words[1] = 0.0; - words[2] = 1.0; - } - // The running sum of squared deviations, which Prometheus starts at NaN for a - // non-finite first sample; then `mean` and the count that says it is seen. - AggregateOp::Stddev | AggregateOp::Stdvar => { - words[0] = if sample.is_finite() { 0.0 } else { f64::NAN }; - words[1] = sample; - words[2] = 1.0; - } - // `value`, `compensation`, `mean`, the incremental flag, then the count. - AggregateOp::Avg => { - words[0] = sample; - words[1] = 0.0; - words[2] = sample; - words[3] = 0.0; - words[4] = 1.0; - } - } +impl FloatState { + fn update(&mut self, op: AggregateOp, sample: f64) { + if !self.seen { + *self = Self { + seen: true, + value: match op { + // The running sum of squared deviations; Prometheus starts it at NaN for a + // non-finite first sample. + AggregateOp::Stddev | AggregateOp::Stdvar if !sample.is_finite() => f64::NAN, + AggregateOp::Stddev | AggregateOp::Stdvar => 0.0, + _ => sample, + }, + mean: sample, + count: 1.0, + ..Self::default() + }; return; } match op { AggregateOp::Sum => { - (words[0], words[1]) = kahan_inc(sample, words[0], words[1]); + (self.value, self.compensation) = kahan_inc(sample, self.value, self.compensation); } - AggregateOp::Avg => update_avg(words, sample), - AggregateOp::Count => words[0] += 1.0, + AggregateOp::Avg => self.update_avg(sample), + AggregateOp::Count => self.count += 1.0, AggregateOp::Max => { - if words[0] < sample || words[0].is_nan() { - words[0] = sample; + if self.value < sample || self.value.is_nan() { + self.value = sample; } } AggregateOp::Min => { - if words[0] > sample || words[0].is_nan() { - words[0] = sample; + if self.value > sample || self.value.is_nan() { + self.value = sample; } } AggregateOp::Group => {} // Welford's update, uncompensated as in Prometheus. AggregateOp::Stddev | AggregateOp::Stdvar => { - let count = words[2] + 1.0; - let delta = sample - words[1]; - words[1] += delta / count; - words[0] += delta * (sample - words[1]); - words[2] = count; + self.count += 1.0; + let delta = sample - self.mean; + self.mean += delta / self.count; + self.value += delta * (sample - self.mean); } } } -} -/// `avg`'s update: sums directly until the running sum would overflow, then switches to an -/// incremental mean, whose flag is the fourth word. -fn update_avg(words: &mut [f64], sample: f64) { - let count = words[4] + 1.0; - words[4] = count; - if words[3] == 0.0 { - let (value, compensation) = kahan_inc(sample, words[0], words[1]); - if !value.is_infinite() { - words[0] = value; - words[1] = compensation; - return; + /// Sums directly until the running sum would overflow, then switches to an incremental mean. + fn update_avg(&mut self, sample: f64) { + self.count += 1.0; + if !self.incremental_mean { + let (value, compensation) = kahan_inc(sample, self.value, self.compensation); + if !value.is_infinite() { + (self.value, self.compensation) = (value, compensation); + return; + } + self.incremental_mean = true; + self.mean = self.value / (self.count - 1.0); + self.compensation /= self.count - 1.0; } - words[3] = 1.0; - words[2] = words[0] / (count - 1.0); - words[1] /= count - 1.0; + let q = (self.count - 1.0) / self.count; + (self.mean, self.compensation) = + kahan_inc(sample / self.count, q * self.mean, q * self.compensation); + } + + fn finish(self, op: AggregateOp) -> Option { + if !self.seen { + return None; + } + Some(match op { + AggregateOp::Sum => self.value + self.compensation, + AggregateOp::Avg if self.incremental_mean => self.mean + self.compensation, + AggregateOp::Avg => self.value / self.count + self.compensation / self.count, + AggregateOp::Count => self.count, + AggregateOp::Min | AggregateOp::Max => self.value, + AggregateOp::Group => 1.0, + AggregateOp::Stdvar => self.value / self.count, + AggregateOp::Stddev => (self.value / self.count).sqrt(), + }) } - let q = (count - 1.0) / count; - (words[2], words[1]) = kahan_inc(sample / count, q * words[2], q * words[1]); } /// Prometheus `kahansum.Inc`, with the Neumaier swap. diff --git a/src/promql/src/aggregate/state.rs b/src/promql/src/aggregate/state.rs index de53c6b9feb..92cb6e70ac1 100644 --- a/src/promql/src/aggregate/state.rs +++ b/src/promql/src/aggregate/state.rs @@ -42,9 +42,9 @@ use datatypes::series::time::{GridSpan, TimeAxis}; use crate::aggregate::{ AggregateOp, BLOCK_POINTS, Counts, FloatAggregator, FloatState, Group, HistogramAccumulator, - HistogramEvidence, LabelKey, accumulator_bytes, block_bytes, btree_backing_bytes, count_series, + HistogramEvidence, LabelKey, accumulator_bytes, btree_backing_bytes, count_series, evidence_summary, finish_groups, finish_groups_budgeted, hash_map_backing_bytes, - histogram_backing_bytes, kahan_inc, label_key, metric_bytes, next_hash_capacity, state_seen, + histogram_backing_bytes, kahan_inc, label_key, metric_bytes, next_hash_capacity, }; use crate::map::validate_float_series; @@ -127,7 +127,7 @@ impl FloatAggregator { // export builds two one-row Arrow batches per point, which the chunk they are handed // to is charged for only once that chunk exists. let (points, payloads) = if group.histograms.is_empty() { - (seen_points(group, width, self.op), 0) + (seen_points(group, width), 0) } else { let payloads = group .histograms @@ -165,21 +165,18 @@ impl FloatAggregator { DataFusionError::Execution("aggregate state is indexed on an evaluated grid".into()) })?; let grid_len = grid.spec().len as usize; - let op = self.op; - let words = op.state_words(); - let point_words = width.saturating_mul(words); let mut groups: Vec = Vec::with_capacity(self.groups.len()); for group in self.groups { let points = if group.histograms.is_empty() { - seen_points(&group, width, op) + seen_points(&group, width) } else { // A field's state is one family, so a group that kept histogram payloads exports // those: a float state beside them would be one column carrying two kinds. - if group.blocks.values().any(|states| { - states - .chunks_exact(words) - .any(|state| state_seen(state, op)) - }) { + if group + .blocks + .values() + .any(|states| states.iter().any(|state| state.seen)) + { return Err(DataFusionError::Execution( "an aggregate field cannot carry both float and histogram state".into(), )); @@ -224,8 +221,8 @@ impl FloatAggregator { continue; } for (&block, block_states) in &group.blocks { - for (offset, point) in block_states.chunks_exact(point_words).enumerate() { - if !point.chunks_exact(words).any(|state| state_seen(state, op)) { + for (offset, point) in block_states.chunks_exact(width).enumerate() { + if !point.iter().any(|state| state.seen) { continue; } let position = block * BLOCK_POINTS + offset; @@ -235,8 +232,8 @@ impl FloatAggregator { ))); } extend_spans(&mut spans, position); - for (field, state) in point.chunks_exact(words).enumerate() { - states[field].push(state_partial(state, op).map(StatePayload::Float)); + for (field, state) in point.iter().enumerate() { + states[field].push(state.partial().map(StatePayload::Float)); } } } @@ -302,14 +299,12 @@ pub(super) fn exported_payload_bytes(accumulator: &HistogramAccumulator) -> usiz } /// The points of `group` that carry a state: the ones a partial of it holds. -fn seen_points(group: &Group, width: usize, op: AggregateOp) -> usize { - let words = op.state_words(); - let point_words = width.saturating_mul(words); +fn seen_points(group: &Group, width: usize) -> usize { group.blocks.values().fold(0usize, |count, states| { count.saturating_add( states - .chunks_exact(point_words) - .filter(|point| point.chunks_exact(words).any(|state| state_seen(state, op))) + .chunks_exact(width) + .filter(|point| point.iter().any(|state| state.seen)) .count(), ) }) @@ -686,10 +681,9 @@ impl FloatMerger { budget: &mut impl FnMut(usize) -> Result<()>, ) -> Result<()> { let width = self.schema.fields.len(); - let op = self.op; - let words = op.state_words(); - let point_words = width.saturating_mul(words); - let states_bytes = block_bytes(width, op); + let block_bytes = BLOCK_POINTS + .saturating_mul(width) + .saturating_mul(size_of::()); let blocks = &mut self.groups[index].blocks; let block_index = point / BLOCK_POINTS; let entries = blocks.len(); @@ -699,24 +693,21 @@ impl FloatMerger { budget( self.retained .saturating_add(live) - .saturating_add(states_bytes) + .saturating_add(block_bytes) .saturating_add(btree_backing_bytes(entries.saturating_add(1))), )?; self.retained = self .retained - .saturating_add(states_bytes) + .saturating_add(block_bytes) .saturating_add(btree_backing_bytes(entries.saturating_add(1))) .saturating_sub(btree_backing_bytes(entries)); - entry.insert(vec![0.0; BLOCK_POINTS * point_words].into_boxed_slice()) + entry.insert(vec![FloatState::default(); BLOCK_POINTS * width].into_boxed_slice()) } }; - let offset = (point % BLOCK_POINTS) * point_words; - for (slot, state) in block[offset..offset + point_words] - .chunks_exact_mut(words) - .zip(states) - { + let offset = (point % BLOCK_POINTS) * width; + for (slot, state) in block[offset..offset + width].iter_mut().zip(states) { if let Some(state) = state { - FloatState::new(slot, op).merge_partial(&state); + slot.merge(self.op, FloatState::from_partial(state)); } } Ok(()) @@ -915,166 +906,97 @@ fn state_at( Ok(Some(state)) } -/// The exported form of the state in `words`, or `None` if it has seen no sample. -/// -/// Only the components the op keeps are filled in; the rest stay at their defaults, exactly as the -/// frame that carries them leaves them out. -fn state_partial(words: &[f64], op: AggregateOp) -> Option { - if !state_seen(words, op) { - return None; +impl FloatState { + fn partial(&self) -> Option { + self.seen.then_some(AccumulatorState { + value: self.value, + mean: self.mean, + compensation: self.compensation, + count: self.count, + incremental_mean: self.incremental_mean, + }) } - Some(match op { - // `value`, its compensation, then the presence flag. - AggregateOp::Sum => AccumulatorState { - value: words[0], - compensation: words[1], - ..Default::default() - }, - // `value`, `compensation`, `mean`, the incremental flag, then the count. - AggregateOp::Avg => AccumulatorState { - value: words[0], - compensation: words[1], - mean: words[2], - incremental_mean: words[3] != 0.0, - count: words[4], - }, - AggregateOp::Count => AccumulatorState { - count: words[0], - ..Default::default() - }, - AggregateOp::Min | AggregateOp::Max => AccumulatorState { - value: words[0], - ..Default::default() - }, - AggregateOp::Group => AccumulatorState::default(), - // `value`, `mean`, then the count. - AggregateOp::Stddev | AggregateOp::Stdvar => AccumulatorState { - value: words[0], - mean: words[1], - count: words[2], - ..Default::default() - }, - }) -} -impl<'a> FloatState<'a> { - /// Writes `other` into this state, for a state that has seen nothing yet. - fn store_partial(self, other: &AccumulatorState) { - let (op, words) = (self.op, self.words); - match op { - AggregateOp::Count => words[0] = other.count, - // Presence alone, whatever the other side keeps: `group` answers 1 either way. - AggregateOp::Group => words[0] = 1.0, - AggregateOp::Min | AggregateOp::Max => { - words[0] = other.value; - words[1] = 1.0; - } - AggregateOp::Sum => { - words[0] = other.value; - words[1] = other.compensation; - words[2] = 1.0; - } - AggregateOp::Stddev | AggregateOp::Stdvar => { - words[0] = other.value; - words[1] = other.mean; - words[2] = other.count; - } - AggregateOp::Avg => { - words[0] = other.value; - words[1] = other.compensation; - words[2] = other.mean; - words[3] = if other.incremental_mean { 1.0 } else { 0.0 }; - words[4] = other.count; - } + fn from_partial(state: AccumulatorState) -> Self { + Self { + seen: true, + value: state.value, + mean: state.mean, + compensation: state.compensation, + count: state.count, + incremental_mean: state.incremental_mean, } } /// Folds the state of another partition into this one. - fn merge_partial(self, other: &AccumulatorState) { - if !state_seen(self.words, self.op) { - self.store_partial(other); + fn merge(&mut self, op: AggregateOp, other: Self) { + if !self.seen { + *self = other; return; } - let (op, words) = (self.op, self.words); match op { AggregateOp::Sum => { - (words[0], words[1]) = - add_compensated((words[0], words[1]), (other.value, other.compensation)); + (self.value, self.compensation) = add_compensated(*self, other); } - AggregateOp::Count => words[0] += other.count, + AggregateOp::Count => self.count += other.count, AggregateOp::Group => {} AggregateOp::Max => { - if words[0] < other.value || words[0].is_nan() { - words[0] = other.value; + if self.value < other.value || self.value.is_nan() { + self.value = other.value; } } AggregateOp::Min => { - if words[0] > other.value || words[0].is_nan() { - words[0] = other.value; + if self.value > other.value || self.value.is_nan() { + self.value = other.value; } } - AggregateOp::Avg => merge_avg(words, other), + AggregateOp::Avg => self.merge_avg(other), AggregateOp::Stddev | AggregateOp::Stdvar => { // Chan et al.'s pairwise update. `delta * (delta * weight)` keeps the product // finite where it is: `delta²` alone overflows for `[0]` and `[1.5e154]`, whose // merged sum of squares is finite. - let count = words[2] + other.count; - let delta = other.mean - words[1]; - let weight = words[2] * other.count / count; - words[1] += delta * (other.count / count); - words[0] = words[0] + other.value + delta * (delta * weight); - words[2] = count; + let count = self.count + other.count; + let delta = other.mean - self.mean; + let weight = self.count * other.count / count; + self.mean += delta * (other.count / count); + self.value = self.value + other.value + delta * (delta * weight); + self.count = count; } } } -} -/// Adds the sums while they stay finite, and otherwise continues with the count-weighted mean, as -/// the serial update switches to an incremental mean once its sum overflows. -/// -/// The state words are `value`, `compensation`, `mean`, the incremental flag and the count. -fn merge_avg(words: &mut [f64], other: &AccumulatorState) { - let count = words[4] + other.count; - if words[3] == 0.0 && !other.incremental_mean { - let (value, compensation) = - add_compensated((words[0], words[1]), (other.value, other.compensation)); - if !value.is_infinite() { - words[0] = value; - words[1] = compensation; - words[4] = count; - return; + /// Adds the sums while they stay finite, and otherwise continues with the count-weighted + /// mean, as the serial update switches to an incremental mean once its sum overflows. + fn merge_avg(&mut self, other: Self) { + let count = self.count + other.count; + if !self.incremental_mean && !other.incremental_mean { + let (value, compensation) = add_compensated(*self, other); + if !value.is_infinite() { + self.value = value; + self.compensation = compensation; + self.count = count; + return; + } } + let (mean, compensation) = self.mean_form(); + let (other_mean, other_compensation) = other.mean_form(); + let (weight, other_weight) = (self.count / count, other.count / count); + (self.mean, self.compensation) = kahan_inc( + other_mean * other_weight, + mean * weight, + compensation * weight + other_compensation * other_weight, + ); + self.incremental_mean = true; + self.count = count; } - let (mean, compensation) = mean_form(words[0], words[1], words[2], words[3] != 0.0, words[4]); - let (other_mean, other_compensation) = mean_form( - other.value, - other.compensation, - other.mean, - other.incremental_mean, - other.count, - ); - let (weight, other_weight) = (words[4] / count, other.count / count); - (words[2], words[1]) = kahan_inc( - other_mean * other_weight, - mean * weight, - compensation * weight + other_compensation * other_weight, - ); - words[3] = 1.0; - words[4] = count; -} -/// The mean and its compensation, converting a sum as the serial update does on switching. -fn mean_form( - value: f64, - compensation: f64, - mean: f64, - incremental: bool, - count: f64, -) -> (f64, f64) { - if incremental { - (mean, compensation) - } else { - (value / count, compensation / count) + /// The mean and its compensation, converting a sum as the serial update does on switching. + fn mean_form(&self) -> (f64, f64) { + if self.incremental_mean { + (self.mean, self.compensation) + } else { + (self.value / self.count, self.compensation / self.count) + } } } @@ -1082,9 +1004,9 @@ fn mean_form( /// Kahan addends: adding the compensation plainly would drop what the compensation already /// holds once later partials cancel the sums, as merging `[1e100, 1]`, `[-1e100, 1e84]` and /// `[-1e84]` would lose the `1`. -fn add_compensated(this: (f64, f64), other: (f64, f64)) -> (f64, f64) { - let (value, compensation) = kahan_inc(other.0, this.0, this.1); - kahan_inc(other.1, value, compensation) +fn add_compensated(this: FloatState, other: FloatState) -> (f64, f64) { + let (value, compensation) = kahan_inc(other.value, this.value, this.compensation); + kahan_inc(other.compensation, value, compensation) } /// Merges `count_values` results of several partitions. Each partial already carries its full diff --git a/src/promql/src/aggregate/tests.rs b/src/promql/src/aggregate/tests.rs index 3cd19c84dab..76f12d99734 100644 --- a/src/promql/src/aggregate/tests.rs +++ b/src/promql/src/aggregate/tests.rs @@ -616,96 +616,6 @@ fn incremental_retained_bytes_match_full_recount_for_many_groups() { } } -/// A state keeps the components its operation folds plus a presence word, which the count already -/// provides where the state has one. That is the whole width of a state, so a `max` state costs two -/// words where an `avg` state costs five instead of every state costing one fixed struct. -#[test] -fn a_state_is_as_wide_as_the_operation_it_belongs_to() { - for (op, words) in [ - (AggregateOp::Count, 1), - (AggregateOp::Group, 1), - (AggregateOp::Min, 2), - (AggregateOp::Max, 2), - (AggregateOp::Sum, 3), - (AggregateOp::Stddev, 3), - (AggregateOp::Stdvar, 3), - (AggregateOp::Avg, 5), - ] { - assert_eq!(op.state_words(), words, "{op:?}"); - let counted = op - .state_fields() - .iter() - .any(|field| matches!(field, StateField::Count)); - assert_eq!( - op.state_words(), - op.state_fields().len() + usize::from(!counted), - "{op:?} keeps its own components and nothing else" - ); - } -} - -/// What a block keeps is what its operation folds: for every operation the fold's retained figure -/// and the recount of its live blocks agree, and a block holds one state per point per field, each -/// as many words as the operation keeps. -#[test] -fn every_operation_recounts_the_states_it_keeps() { - let schema = schema_with_grid_len(&["value", "other"], 256); - for op in [ - AggregateOp::Sum, - AggregateOp::Count, - AggregateOp::Avg, - AggregateOp::Min, - AggregateOp::Max, - AggregateOp::Group, - AggregateOp::Stddev, - AggregateOp::Stdvar, - ] { - let mut aggregator = - FloatAggregator::try_new(&schema, op, Grouping::By(&["zone"])).unwrap(); - assert_eq!( - aggregator.state_bytes(), - aggregator.recounted_state_bytes(), - "{op:?} initial state" - ); - let updates = [ - input( - &schema, - metric("m", &[("zone", "a")]), - 0, - &[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]], - ), - input( - &schema, - metric("m", &[("zone", "a")]), - 64, - &[vec![Some(4.0)], vec![Some(5.0)]], - ), - input( - &schema, - metric("m", &[("zone", "b")]), - 128, - &[vec![Some(6.0)], vec![Some(7.0)]], - ), - ]; - for (index, series) in updates.iter().enumerate() { - aggregator.update(series).unwrap(); - assert_eq!( - aggregator.state_bytes(), - aggregator.recounted_state_bytes(), - "{op:?} update {index}" - ); - } - // Three blocks of two fields, each state as wide as the operation folds it. - let blocks = &aggregator.groups[0].blocks; - assert_eq!(blocks.len(), 2, "{op:?} blocks the first group wrote"); - assert_eq!( - blocks.values().map(|block| block.len()).sum::(), - blocks.len() * BLOCK_POINTS * schema.fields.len() * op.state_words(), - "{op:?} words per block" - ); - } -} - /// The unbudgeted entry point is the budgeted one with a no-op budget, so this compares one path /// with itself: it is a smoke check that the wrapper still runs, not evidence that the kernels are /// right. The numbers are checked against Prometheus in `aggregates_match_prometheus` and @@ -769,7 +679,9 @@ fn adjacent_blocks_are_admitted_and_accounted_separately() { assert_eq!(aggregator.groups[0].blocks.len(), 2); // A new block is admitted on the retained state alone, so the last admission settles at // exactly what the fold keeps and no staging buffer is charged for the samples in it. - let block = block_bytes(schema.fields.len(), AggregateOp::Sum); + let block = BLOCK_POINTS + .saturating_mul(schema.fields.len()) + .saturating_mul(std::mem::size_of::()); let (first, second) = ( admissions[admissions.len() - 2], admissions[admissions.len() - 1], @@ -798,7 +710,9 @@ fn a_rejected_block_allocation_does_not_insert_a_block() { // The figure a new block's admission adds: the block's states and its tree node, on top of // whatever the fold already holds. Located by the value, not by which call reaches it, so // reordering the earlier admissions cannot make this test pass without testing anything. - let block_states = block_bytes(schema.fields.len(), AggregateOp::Sum); + let block_states = BLOCK_POINTS + .saturating_mul(schema.fields.len()) + .saturating_mul(std::mem::size_of::()); let block = block_states.saturating_add(btree_backing_bytes(1)); let mut aggregator = FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap(); @@ -842,7 +756,9 @@ fn a_rejected_later_block_keeps_the_blocks_the_group_already_holds() { // The group and the tree node are already there, so the block's own states are the whole // charge of the second block. let later = input(&schema, metric("m", &[]), 64, &[vec![Some(2.0)]]); - let block_states = block_bytes(schema.fields.len(), AggregateOp::Sum); + let block_states = BLOCK_POINTS + .saturating_mul(schema.fields.len()) + .saturating_mul(std::mem::size_of::()); let threshold = retained .saturating_add(btree_backing_bytes(2).saturating_sub(btree_backing_bytes(1))) .saturating_add(block_states); @@ -948,7 +864,9 @@ fn the_fold_releases_the_series_workspace_before_it_admits_a_block() { let state = aggregator.state_bytes(); assert_eq!(aggregator.recounted_state_bytes(), state); - let block = block_bytes(schema.fields.len(), AggregateOp::Sum); + let block = BLOCK_POINTS + .saturating_mul(schema.fields.len()) + .saturating_mul(std::mem::size_of::()); let entry = state .saturating_sub(block) .saturating_sub(btree_backing_bytes(1)); @@ -967,105 +885,6 @@ fn the_fold_releases_the_series_workspace_before_it_admits_a_block() { ); } -/// A chunk admits its workspace and the group it may open in one pool call, releases the -/// workspace at the fold entry, and pays for each block it opens: three calls for a chunk that -/// falls in one block, four for one that opens two. -/// -/// The fold used to admit the workspace, then a wider bound of it, then the grouping and its key, -/// then the new group on top of them: six admissions for a chunk that opens a group and one block, -/// seven for one that opens two, on top of the runtime's own resize once the chunk is folded. -#[test] -fn a_chunk_admits_its_workspace_release_and_blocks_in_three_or_four_calls() { - let schema = schema(&["value", "other"]); - let one_block = input( - &schema, - metric("m", &[("zone", "a")]), - 0, - &[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]], - ); - // Two samples of the first field straddle the edge at 64, so the fold opens a second block. - let two_blocks = input( - &schema, - metric("m", &[("zone", "a")]), - 63, - &[vec![Some(1.0), Some(2.0)], vec![Some(3.0)]], - ); - let admissions = |series: &Series| { - let mut aggregator = - FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone"])).unwrap(); - let mut calls = Vec::new(); - aggregator - .update_budgeted(series, |bytes| { - calls.push(bytes); - Ok(()) - }) - .unwrap(); - assert_eq!(aggregator.state_bytes(), aggregator.recounted_state_bytes()); - calls - }; - - let one = admissions(&one_block); - assert_eq!( - one.len(), - 3, - "the workspace, the release and one block: {one:?}" - ); - assert!( - one[0] > one[1], - "the workspace was admitted beside the retained state: {one:?}" - ); - let block = block_bytes(schema.fields.len(), AggregateOp::Sum); - assert_eq!( - one[2], - one[1] - .saturating_add(block) - .saturating_add(btree_backing_bytes(1)), - "the block is admitted on the retained state alone" - ); - - let two = admissions(&two_blocks); - assert_eq!( - two.len(), - 4, - "the workspace, the release and two blocks: {two:?}" - ); - assert!(two[3] > two[2], "each block is admitted separately"); -} - -/// A chunk that only writes into states its group already holds costs the workspace admission and -/// the release, and no block. -#[test] -fn a_chunk_of_existing_blocks_admits_only_its_workspace_and_release() { - let schema = schema(&["value"]); - let mut aggregator = - FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&["zone"])).unwrap(); - aggregator - .update(&input( - &schema, - metric("m", &[("zone", "a")]), - 0, - &[vec![Some(1.0)]], - )) - .unwrap(); - let mut calls = Vec::new(); - aggregator - .update_budgeted( - &input( - &schema, - metric("m", &[("zone", "a")]), - 1, - &[vec![Some(2.0)]], - ), - |bytes| { - calls.push(bytes); - Ok(()) - }, - ) - .unwrap(); - assert_eq!(calls.len(), 2, "the workspace and the release: {calls:?}"); - assert_eq!(aggregator.state_bytes(), aggregator.recounted_state_bytes()); -} - /// A block a field holds no value in is not allocated: two fields whose samples fall in different /// blocks leave the group with the blocks they write, not the blocks between them. #[test] @@ -2164,7 +1983,7 @@ fn merge_charges_one_block_per_new_block_not_per_point() { aggregator.update(&series).unwrap(); let partial = aggregator.finish_partial().unwrap(); assert_eq!(partial.groups[0].states[0].len(), 64); - let block = block_bytes(64, AggregateOp::Sum); + let block = BLOCK_POINTS * 64 * std::mem::size_of::(); let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); let (peak, result) = peak_admission(|budget| merge_partial_budgeted(&mut merger, partial, budget)); diff --git a/src/promql/src/extension_plan.rs b/src/promql/src/extension_plan.rs index fc619cd7047..71674ac5fbf 100644 --- a/src/promql/src/extension_plan.rs +++ b/src/promql/src/extension_plan.rs @@ -44,8 +44,8 @@ pub use empty_metric::{EmptyMetric, EmptyMetricExec, EmptyMetricStream, build_sp pub use fused_series::is_canonical_producer_exec; pub use fused_series::{ FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec, - FusedSeriesStream, InputLocality, input_locality, keys_survive_metadata_drop, - matching_operands_drop_key, tree_source_fields, + FusedSeriesStream, InputLocality, input_locality, matching_operands_drop_key, + tree_source_fields, }; pub use histogram_fold::{ HistogramFold, HistogramFoldExec, HistogramFoldOperation, HistogramFoldStream, diff --git a/src/promql/src/extension_plan/fused_series.rs b/src/promql/src/extension_plan/fused_series.rs index c3305d5567f..7774117b51c 100644 --- a/src/promql/src/extension_plan/fused_series.rs +++ b/src/promql/src/extension_plan/fused_series.rs @@ -78,7 +78,6 @@ use datafusion::physical_plan::{ Partitioning, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datatypes::data_type::DataType as _; -use datatypes::schema::ColumnSchema; use datatypes::series::SeriesOrdering; use datatypes::series::frame::{ FrameContent, FrameLabelMode, SeriesFrameDeclaration, aggregate_state_frame_schema, @@ -97,7 +96,7 @@ use crate::extension_plan::series_repartition::SeriesRepartitionExec; use crate::extension_plan::{RawSeriesSourceExec, SeriesRouting}; use crate::grid_series::METADATA_LABELS; use crate::histogram_eval::HISTOGRAM_ROUTING_EXCLUDED_LABELS; -use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor, SourceFacts}; +use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor}; use crate::native::metrics::{NodeMetricsSnapshot, is_node_metric}; use crate::native::plan::{ AggregateStage, BoundMap, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree, @@ -991,7 +990,6 @@ impl FusedSeries { output_schema, self.binding.time_index.clone(), tags, - self.binding.metric_name.clone(), self.format, target_partitions, self.canonical_producer, @@ -2127,13 +2125,6 @@ pub struct FusedSeriesExec { time_column: String, /// The tag column names, indexed by [`OutputColumn::Tag`]. tags: Arc<[String]>, - /// The one metric every input series is named, taken from the binding. - /// - /// The canonical pass judged a dropped label set by this: a node that drops the metric name is - /// only told to stream when everything below it is named the same one. The execution reads it - /// so the collection it performs at run time is decided by that same proof; `None` is a binding - /// that names no metric - one raw source per input - and every consumer then collects. - metric_name: Option>, /// The columns this node produces. output_schema: SchemaRef, /// The evidence claimed by this plan's output partitions. @@ -2181,42 +2172,20 @@ struct StoredSeriesLabels { /// Every tag name a stored series reports besides its metric name. tags: Arc<[Arc]>, /// Whether the primary key keeps telling two stored series apart once a name-dropping kernel - /// has normalized their labels; see [`keys_survive_metadata_drop`] for the rule itself. + /// has normalized their labels. + /// + /// Two facts make that true. The tags and the primary key must be free of the metadata labels: + /// `__type__` and `__unit__` go with the metric name when a kernel drops it, so a binding that + /// keys a series by either of them stops telling two series apart once the name is gone. And + /// every primary-key column must always decode a value, which is what a non-nullable column + /// states. A kernel that drops the name drops every label whose value is empty with it, and a + /// nullable column lets one series hold `NULL` while another holds `''`: two different storage + /// keys, which decode to an absent label and to an empty one, so both are handed over as no + /// label at all once the empty one is dropped. Two such series that agree on every other tag + /// then leave as one identity, whether they arrive in one partition or in two. keys_survive_metadata_drop: bool, } -/// Whether the storage key of a scan keeps telling its stored series apart once a name-dropping -/// kernel has normalized their labels. -/// -/// `logical_tags` are the tag names every stored series of the scan reports, and `key_columns` are -/// the columns the scan keys one series by. Two facts make the drop injective, and both are read -/// from the scan's own binding rather than from a tree above it: -/// -/// - Neither the tags nor the key columns may be a metadata label. `__type__` and `__unit__` go with -/// the metric name when a kernel drops it, so a scan that keys a series by either of them stops -/// telling two series apart once the name is gone. -/// - Every key column must always decode a value, which is what a non-nullable column states. A -/// kernel that drops the name drops every label whose value is empty with it, and a nullable -/// column lets one series hold `NULL` while another holds `''`: two different storage keys, which -/// decode to an absent label and to an empty one, so both are handed over as no label at all once -/// the empty one is dropped. Two such series that agree on every other tag then leave as one -/// identity, whether they arrive in one partition or in two. -/// -/// This is the rule the logical canonical pass applies to decide whether a node that drops the -/// metadata labels has to be collected, and the one the physical per-series scan applies to decide -/// whether such a node can stream. Both read it here so the two answers cannot drift apart: the -/// planner reads it from the resolved table's metadata, the execution from the bound scan. -pub fn keys_survive_metadata_drop<'a, 'b>( - logical_tags: impl IntoIterator, - key_columns: impl IntoIterator, -) -> bool { - let metadata = |name: &str| METADATA_LABELS.contains(&name); - logical_tags.into_iter().all(|tag| !metadata(tag)) - && key_columns - .into_iter() - .all(|column| !metadata(&column.name) && !column.is_nullable()) -} - impl StoredSeriesLabels { /// The labels `label_source` proves about every stored series it reads. fn of(label_source: &RawLabelSource) -> Option { @@ -2230,17 +2199,19 @@ impl StoredSeriesLabels { else { return None; }; + let metadata = |name: &str| METADATA_LABELS.contains(&name); Some(Self { tags: logical_tags .iter() .map(|tag| Arc::clone(&tag.name)) .collect(), - keys_survive_metadata_drop: keys_survive_metadata_drop( - logical_tags.iter().map(|tag| tag.name.as_ref()), - physical_metadata - .primary_key_columns() - .map(|column| &column.column_schema), - ), + keys_survive_metadata_drop: logical_tags.iter().all(|tag| !metadata(&tag.name)) + && physical_metadata.primary_key_columns().all(|column| { + // A nullable key column can be `NULL` in one series and `''` in another, and a + // name-dropping kernel turns both into no label; only a column that always + // decodes a value keeps the two apart under such a kernel. + !metadata(&column.column_schema.name) && !column.column_schema.is_nullable() + }), }) } @@ -2519,7 +2490,6 @@ impl FusedSeriesExec { output_schema, time_column, tags, - None, FusedOutputFormat::PlainRows, None, false, @@ -2534,7 +2504,6 @@ impl FusedSeriesExec { output_schema: SchemaRef, time_column: String, tags: Vec, - metric_name: Option>, format: FusedOutputFormat, matching_parallelism: Option, canonical_producer: bool, @@ -2818,7 +2787,6 @@ impl FusedSeriesExec { classification: classification.into(), time_column, tags: tags.into(), - metric_name, output_schema, duplicate_evidence: Mutex::new(EvidenceClaims { evidence: None, @@ -2970,7 +2938,6 @@ impl ExecutionPlan for FusedSeriesExec { self.output_schema.clone(), self.time_column.clone(), self.tags.to_vec(), - self.metric_name.clone(), self.format, self.matching_parallelism, self.canonical_producer, @@ -3003,19 +2970,10 @@ impl ExecutionPlan for FusedSeriesExec { evidence: Arc::clone(evidence), active: true, }); - // The canonical pass decided where a merge belongs from the labels the binding names; the - // execution judges the collection it performs by the same fact, so an already-streaming - // window is not turned into a collecting one here. - let sources = SourceFacts::uniform( - self.tags.to_vec(), - self.metric_name.clone(), - self.series_schemas.len(), - ); - let mut state = match NativeExecutionState::try_new_with_sources( + let mut state = match NativeExecutionState::try_new( &self.tree, Arc::clone(&context.runtime_env().memory_pool), evidence.clone(), - &sources, ) { Ok(state) => state, Err(error) => { diff --git a/src/promql/src/extension_plan/fused_series/tests.rs b/src/promql/src/extension_plan/fused_series/tests.rs index 182270b0067..9b5c7a02000 100644 --- a/src/promql/src/extension_plan/fused_series/tests.rs +++ b/src/promql/src/extension_plan/fused_series/tests.rs @@ -238,24 +238,6 @@ fn window_tree_of(function: RangeFunction) -> NativeTree { NativeTree::try_new(root).expect("a window function keeps the raw input and its grid") } -/// `rate(x[5m])` as the canonical pass hands it over: one merge over the window's output. -/// -/// A range function drops the metric name, so two series that agreed on every other label leave the -/// window under one label set. Collecting them is what refuses a second value at one position: the -/// merge groups by the label set the window handed out and reports the collision, as Prometheus -/// refuses two series of one label set. -fn canonical_rate_tree_of(schema: Arc) -> NativeTree { - let window = rate_tree_of(schema); - let output = window.root().output.clone(); - NativeTree::try_new(NativeNode { - expr: NativeExpr::Merge { - input: Box::new(window.root().clone()), - }, - output, - }) - .expect("a merge over a window on a grid is a legal tree") -} - /// `left + right` on one grid, both operands reading the one actual source. fn binary_tree(left_schema: Arc, right_schema: Arc) -> NativeTree { binary_tree_over_sources(BinaryOp::Add, left_schema, right_schema, 0, 0) @@ -1963,11 +1945,7 @@ fn zone_output_schema() -> SchemaRef { /// `rate(x[5m])` over frames that carry `zone` as well as `host`, with both in the output. fn zone_node() -> FusedSeries { - zone_node_with(rate_tree()) -} - -/// The same node over `tree`, so a test can stage the tree the canonical pass produces. -fn zone_node_with(tree: NativeTree) -> FusedSeries { + let tree = rate_tree(); let binding = binding_of( &tree, raw_schema(), @@ -3419,7 +3397,7 @@ fn rows(batches: &[RecordBatch]) -> Vec { const DUPLICATE_LABEL_SET: &str = "vector cannot contain metrics with the same labelset {host=\"web-1\"}"; -/// Two raw series that are already one Prometheus identity are refused by the merge that would +/// Two raw series that are already one Prometheus identity are refused by the runtime that would /// have to answer both. /// /// `{host="web-1", zone=""}` and `{host="web-1"}` are one identity to Prometheus: an empty label @@ -3427,16 +3405,12 @@ const DUPLICATE_LABEL_SET: &str = /// What the two *raw* keys differ in is the empty `zone` against the absent one, which the scan /// encodes differently and the decoder therefore opens two input series for. `rate` drops the /// metric name and the empty value, so the second raw series arrives on the canonical label set the -/// first one already occupies, and the merge collecting the window's output refuses it: the -/// refusal names that label set rather than answering it, because answering would report one point -/// twice. +/// first one already occupies; the refusal names that label set rather than answering it, because +/// answering would report one point twice. #[tokio::test] async fn differing_raw_keys_that_close_on_one_label_set_are_refused() { - let exec = plan_of( - zone_node_with(canonical_rate_tree_of(raw_schema())), - memory_exec(frames(&zone_chunks())), - ) - .expect("the runtime executes a canonical rate tree"); + let exec = plan_of(zone_node(), memory_exec(frames(&zone_chunks()))) + .expect("the runtime executes a rate tree"); let error = collect(exec, SessionContext::default().task_ctx()) .await .expect_err("the second raw key cannot be answered under the first one's label set"); @@ -3450,25 +3424,13 @@ async fn differing_raw_keys_that_close_on_one_label_set_are_refused() { /// /// The decoder holds the batch it is reading, the chunks it decoded from it and its own source, so a /// stream that kept its input after the error would hold all three until the consumer drops it. The -/// error staged here is the truncated second series: the first one's rows are handed out, and the -/// second one ends the input without saying it was its last chunk, which the frame decoder refuses. +/// collision above is the error: it happens on the second series, after the first one's rows have +/// been handed out. #[tokio::test] async fn a_runtime_error_after_rows_releases_the_input() { let dropped = Arc::new(AtomicBool::new(false)); let schema = frame_schema(); - // The second series is cut off: its chunk does not close it. - let mut chunks = zone_chunks(); - chunks.pop().expect("the collision stages two series"); - let samples = series_samples(1.0); - chunks.push(chunk_of( - &raw_schema(), - "zone-absent", - metric(&[(HOST, "web-1")], Some(METRIC)), - ×_of(&samples), - vec![values_of(&samples)], - false, - )); - let batches = frames(&chunks); + let batches = frames(&zone_chunks()); let source = staged(schema.clone(), { let dropped = Arc::clone(&dropped); move || Box::pin(StagedStream::new(schema.clone(), batches.clone()).dropped(&dropped)) @@ -3490,7 +3452,7 @@ async fn a_runtime_error_after_rows_releases_the_input() { } }; assert!( - error.to_string().contains("ended before key was final"), + error.to_string().contains(DUPLICATE_LABEL_SET), "refused for another reason: {error}" ); // The first series' rows were handed out before the error, and the input is gone although the @@ -3505,17 +3467,16 @@ async fn a_runtime_error_after_rows_releases_the_input() { ); } -/// A collision produced inside one native execution is refused by the merge that collects it. +/// A collision produced inside one native execution is still refused by its local runtime state. /// /// This is the collision the canonical label set exists for, at the layer that sees it: /// `http_requests_total{host="web-1"}` and `http_errors_total{host="web-1"}` reach the window as two /// identities and leave it as `{host="web-1"}`. [`FusedSeries`] reads one metric per scan, so this /// collision cannot be staged through it; the runtime is driven directly, with the names on the raw -/// chunks. The window is handed over collected, as the canonical pass places it: nothing is answered -/// until its input ends, and the merge then refuses the second value of the one label set. +/// chunks. #[test] fn two_differently_named_series_collide_once_rate_drops_the_name() { - let tree = canonical_rate_tree_of(raw_schema()); + let tree = rate_tree(); let mut execution = NativeExecutor::new( NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None) .expect("the runtime executes a rate tree"), @@ -3534,6 +3495,13 @@ fn two_differently_named_series_collide_once_rate_drops_the_name() { ), ) .expect("the first series is a legal raw chunk"); + assert!( + execution + .next_chunk() + .expect("the first series runs") + .is_some(), + "the first series is answered" + ); execution .push_raw( InputId::new(0), @@ -3547,11 +3515,8 @@ fn two_differently_named_series_collide_once_rate_drops_the_name() { ), ) .expect("the second series is a legal raw chunk"); - // Both series sit in the merge's collection: the second one puts a value where the first one - // already holds one, and the merge refuses the pair as it collects them. let error = execution - .end_of_input() - .and_then(|()| execution.next_chunk().map(|_| ())) + .next_chunk() .expect_err("the second series puts a value where the first one already holds one"); assert!( error.to_string().contains(DUPLICATE_LABEL_SET), diff --git a/src/promql/src/extension_plan/series_repartition.rs b/src/promql/src/extension_plan/series_repartition.rs index 974ff75287c..7b23e22aa2c 100644 --- a/src/promql/src/extension_plan/series_repartition.rs +++ b/src/promql/src/extension_plan/series_repartition.rs @@ -94,8 +94,6 @@ //! until its per-input close/EOF barrier is satisfied, then //! reserve merge workspace for that identity alone. This bounds merge workspace to one pending //! identity, while its inputs remain charged until handoff. No task waits for memory admission. -//! An output registers one consumer with the pool and splits every identity's charge off it, so the -//! pool holds an entry per output stream rather than one per series that stream routes. //! //! An output ends only when every input partition finished and its queue was drained; a chain that //! closed locally is not the end of anything. Each input sends its own EOF to each live output before @@ -2109,14 +2107,6 @@ struct RepartitionStream { inputs: usize, output: VecDeque, handoff: Arc>, - /// What every identity of this output is charged to. - /// - /// One registration for the whole output stream: each identity's descriptor, each owner's - /// merge workspace and each raw owner's reconstruction are reservations split off it, so the - /// pool holds an entry per output partition instead of one per series the output routes. A - /// reservation split off it still keeps its bytes live until it is dropped, and the pool sees - /// each of those bytes exactly once. - groups_reservation: MemoryReservation, /// Whether a new pass over the pending identities is owed after an input EOF. /// /// A chain can only complete the identity it carries, so the chain arm checks that group @@ -2200,7 +2190,6 @@ impl RepartitionStream { inputs, output: VecDeque::new(), handoff: Arc::new(Mutex::new(Handoff::default())), - groups_reservation: MemoryConsumer::new("SeriesRepartitionGroups").register(pool), rescan: false, resume_after: None, reported: false, @@ -2218,7 +2207,7 @@ impl RepartitionStream { let rows = group.fragments.iter().fold(0usize, |rows, series| { rows.saturating_add(series.time.len()) }); - let reservation = self.groups_reservation.new_empty(); + let reservation = MemoryConsumer::new("SeriesRepartitionOwner").register(&self.pool); reservation.try_grow(merge_workspace_bytes( rows, group.fragments.len(), @@ -2292,7 +2281,7 @@ impl RepartitionStream { // Reconstruction's working set depends on how the pieces interleave, which is only known // while it runs: charge each allocation as it is made rather than reserving a worst case // that a well-ordered identity never approaches. - let workspace = self.groups_reservation.new_empty(); + let workspace = MemoryConsumer::new("SeriesRepartitionRawOwner").register(&self.pool); let mut reserve = |bytes: usize| workspace.try_grow(bytes); let mut shrink = |bytes: usize| { let result = workspace.try_shrink(bytes); @@ -2491,7 +2480,8 @@ impl Stream for RepartitionStream { } }; if !this.groups.contains_key(&key) { - let metadata = this.groups_reservation.new_empty(); + let metadata = + MemoryConsumer::new("SeriesRepartitionIdentity").register(&this.pool); let key_bytes = match &key { OwnerKey::Evaluated((name, tags)) => { diff --git a/src/promql/src/extension_plan/series_repartition/tests.rs b/src/promql/src/extension_plan/series_repartition/tests.rs index ec4efae6b4c..7e4e150931e 100644 --- a/src/promql/src/extension_plan/series_repartition/tests.rs +++ b/src/promql/src/extension_plan/series_repartition/tests.rs @@ -521,77 +521,13 @@ impl ExecutionPlan for Scripted { /// A task context whose memory pool is `pool`, so what the node charges can be read afterwards. fn task_context(pool: &Arc) -> Arc { - context_over(Arc::clone(pool) as Arc) -} - -/// A task context over `pool`, for a test that keeps its own pool to read it afterwards. -fn context_over(pool: Arc) -> Arc { let runtime = RuntimeEnvBuilder::new() - .with_memory_pool(pool) + .with_memory_pool(Arc::clone(pool) as Arc) .build() .expect("the runtime environment builds"); Arc::new(TaskContext::default().with_runtime(Arc::new(runtime))) } -/// A pool that counts the consumers registered with it, so a test can see how many entries the node -/// adds whatever the input carries. -#[derive(Debug)] -struct RegistrationCountingPool { - inner: GreedyMemoryPool, - registrations: AtomicUsize, -} - -impl RegistrationCountingPool { - fn new(bytes: usize) -> Arc { - Arc::new(Self { - inner: GreedyMemoryPool::new(bytes), - registrations: AtomicUsize::new(0), - }) - } - - /// How many consumers were registered with the pool, in total. - fn registrations(&self) -> usize { - self.registrations.load(Ordering::SeqCst) - } -} - -impl fmt::Display for RegistrationCountingPool { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "registration counting test pool") - } -} - -impl MemoryPool for RegistrationCountingPool { - fn name(&self) -> &str { - "registration counting test pool" - } - - fn register(&self, consumer: &MemoryConsumer) { - self.registrations.fetch_add(1, Ordering::SeqCst); - self.inner.register(consumer); - } - - fn unregister(&self, consumer: &MemoryConsumer) { - self.inner.unregister(consumer); - } - - fn grow(&self, reservation: &MemoryReservation, additional: usize) { - self.inner.grow(reservation, additional); - } - - fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { - self.inner.shrink(reservation, shrink); - } - - fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> DataFusionResult<()> { - self.inner.try_grow(reservation, additional) - } - - fn reserved(&self) -> usize { - self.inner.reserved() - } -} - /// The node over `child` routing into `outputs` partitions. fn node(child: Arc, outputs: usize) -> Arc { Arc::new(SeriesRepartitionExec::try_new(child, outputs).expect("the node is built")) @@ -2756,42 +2692,6 @@ async fn single_fragment_large_identity_fits_finite_payload_budget() { ); } -/// Each identity used to register a pool consumer of its own, so the pool's bookkeeping grew with -/// the number of series an output routed. The stream registers once per output stream now, and the -/// identity's descriptor, an owner's merge workspace and a raw owner's reconstruction are -/// reservations split off it: more identities must not mean more entries, and the bytes each of -/// them holds are still released. -#[tokio::test] -async fn pool_entries_follow_output_streams_not_identities() { - let registrations = |identities: usize| async move { - let steps: Vec = (0..identities) - .map(|index| Step::chunk(chunk(labels(&format!("h{index:03}")), 1, 1, true))) - .collect(); - let pool = RegistrationCountingPool::new(1 << 20); - let context = context_over(Arc::clone(&pool) as Arc); - let plan: Arc = node(Scripted::plan(vec![steps]), 1); - let outputs = drain(&plan, &context).await; - let chunks = outputs - .into_iter() - .flat_map(|output| output.expect("the output routes every identity")) - .collect::>(); - assert_eq!(identities, chunks.len(), "each identity is published once"); - assert_eq!(0, pool.reserved(), "every charge is released with the read"); - pool.registrations() - }; - - let few = registrations(1).await; - let many = registrations(16).await; - assert_eq!( - few, many, - "the entries the node registers follow the identities: {few} for one and {many} for sixteen" - ); - assert!( - few <= 4, - "{few} pool entries for one input and one output stream" - ); -} - /// An input EOF already consumed still counts when a different input first contributes an identity. #[tokio::test] async fn eof_before_first_identity_contribution_makes_group_ready() { @@ -4288,83 +4188,6 @@ async fn owner_rejects_conflicting_values_at_same_identity_and_position() { ); } -/// The owner merges the histogram pieces of one label set that its inputs hold. -/// -/// A canonical boundary hands one label set to a single owner, and that owner is where the pieces the -/// producer's partitions collected meet: here two inputs hold complementary positions of one label -/// set and each piece carries its own histogram payload, so the merge rebuilds one field over the -/// union of the positions with every row still naming the payload it came from. The pieces are closed -/// as they arrive, so the merged series is also what proves the closing chunk of a chain survives the -/// merge. -#[tokio::test] -async fn owner_merges_histogram_pieces_of_one_label_set_across_inputs() { - let metric = labels("histogram-pieces"); - let (first, closing) = histogram_rows_chunk_with_sum(metric.clone(), 0, 1, 10.0); - let (second, _) = histogram_rows_chunk_with_sum(metric.clone(), 8, 1, 20.0); - let child = Scripted::plan(vec![ - vec![Step::Chunk(first, closing)], - vec![Step::chunk((second, true))], - ]); - let plan: Arc = node(child, 2); - let outputs = drain(&plan, &SessionContext::default().task_ctx()).await; - let owner = Routing::new(2).output(&metric); - let chunks = outputs[owner].as_ref().expect("the owner output decodes"); - assert_output_contract(chunks, &[(metric.clone(), vec![0, 8])]); - assert_eq!(1, chunks.iter().filter(|(_, closed)| *closed).count()); - // One row per valued position, in order, each holding the payload of the piece that pushed it: a - // merge that rebuilt the field without reindexing the payloads would answer one payload twice. - let merged = chunks - .iter() - .flat_map(|(series, _)| { - let field = series.fields.get(0).expect("one field"); - series.time.positions().map(move |(position, row)| { - let datatypes::series::samples::SampleRef::Histogram(view) = field.sample(row) - else { - panic!("position {position} of the merged series is not a histogram"); - }; - (position, view.sum, view.count) - }) - }) - .collect::>(); - assert_eq!( - vec![(0, 10.0, 4.0), (8, 20.0, 4.0)], - merged, - "the merged histograms are not the ones the pieces carried" - ); -} - -/// Two histogram pieces of one label set at one position fail the merge. -/// -/// The pieces were routed to one owner because they are one label set, so a position both of them -/// fill would carry two values of one field. The kernel reports that the way it does for floats, and -/// the histogram payload makes no difference to it. -#[tokio::test] -async fn owner_rejects_two_histogram_pieces_at_one_position() { - let metric = labels("histogram-conflict"); - let (first, _) = histogram_rows_chunk_with_sum(metric.clone(), 4, 1, 10.0); - let (second, _) = histogram_rows_chunk_with_sum(metric.clone(), 4, 1, 20.0); - let child = Scripted::plan(vec![ - vec![Step::chunk((first, true))], - vec![Step::chunk((second, true))], - ]); - let plan: Arc = node(child, 1); - let outputs = drain(&plan, &SessionContext::default().task_ctx()).await; - let error = outputs[0] - .as_ref() - .err() - .unwrap_or_else(|| { - panic!( - "two histogram pieces of one label set at one position were merged: {:?}", - outputs[0].as_ref().map(|chunks| summaries(chunks)) - ) - }) - .to_string(); - assert!( - error.contains("cannot contain metrics with the same labelset"), - "the failure does not name the shared label set: {error}" - ); -} - /// Same-identity input chunks are collected and emitted as one closed series. #[tokio::test] async fn same_canonical_key_chains_merge_into_one_series() { diff --git a/src/promql/src/grid_series.rs b/src/promql/src/grid_series.rs index d3f754a7335..532283528dd 100644 --- a/src/promql/src/grid_series.rs +++ b/src/promql/src/grid_series.rs @@ -233,7 +233,13 @@ impl GridSeriesBuilder { for (field, value) in self.fields.iter_mut().zip(values) { match value { Some(GridSample::Histogram(h)) => { - let column = field.start_histogram(); + field.values.push(0.0); + field.nulls.append(false); + let rows = field.values.len() - 1; + let column = field.histograms.get_or_insert_with(|| HistogramColumn { + batch: HistogramBatchBuilder::new(), + indices: vec![None; rows], + }); column.indices.push(Some(h.push_to(&mut column.batch)?)); } Some(GridSample::Float(value)) => field.push_float(Some(*value)), @@ -243,38 +249,6 @@ impl GridSeriesBuilder { Ok(()) } - /// Like [`GridSeriesBuilder::push_samples`], with the values taken as the fields hand them out. - /// - /// A histogram value goes into its column straight from the view it was read as: the owned - /// form beside this builder exists for the paths that compute a histogram, while a selection - /// that only hands a sample on has nothing to compute and copies nothing. The row this writes - /// and the row [`GridSeriesBuilder::push_samples`] writes are the same, so a value pushed - /// either way leaves the same payload. - pub(crate) fn push_refs(&mut self, index: usize, values: &[SampleRef<'_>]) -> Result<()> { - if !self.start_row( - index, - values.len(), - values - .iter() - .all(|value| matches!(value, SampleRef::Missing)), - )? { - return Ok(()); - } - for (field, value) in self.fields.iter_mut().zip(values) { - match value { - SampleRef::Histogram(view) => { - let column = field.start_histogram(); - column - .indices - .push(Some(column.batch.push(*view).map_err(external)?)); - } - SampleRef::Float(value) => field.push_float(Some(*value)), - SampleRef::Missing => field.push_float(None), - } - } - Ok(()) - } - /// Validates the row and retains it; returns whether it holds a value to write. fn start_row(&mut self, index: usize, len: usize, empty: bool) -> Result { if len != self.schema.fields.len() { @@ -430,18 +404,6 @@ impl FieldBuilder { column.indices.push(None); } } - - /// Writes the value columns of a row holding a histogram, opening the histogram column if this - /// is its first value, and hands back the column the new row's index still goes into. - fn start_histogram(&mut self) -> &mut HistogramColumn { - self.values.push(0.0); - self.nulls.append(false); - let rows = self.values.len() - 1; - self.histograms.get_or_insert_with(|| HistogramColumn { - batch: HistogramBatchBuilder::new(), - indices: vec![None; rows], - }) - } } /// Prometheus `schema.IsMetadataLabel`: the labels removed with the metric name. diff --git a/src/promql/src/grid_series/tests.rs b/src/promql/src/grid_series/tests.rs index 203de9e9f2c..98c65bcb16f 100644 --- a/src/promql/src/grid_series/tests.rs +++ b/src/promql/src/grid_series/tests.rs @@ -16,9 +16,7 @@ use std::mem::size_of; use std::sync::Arc; use datafusion::arrow::array::{Float64Array, UInt32Array}; -use datatypes::series::histogram::{ - CounterResetHint, HistogramBatch, HistogramBatchBuilder, HistogramView, -}; +use datatypes::series::histogram::{HistogramBatch, HistogramBatchBuilder}; use datatypes::series::samples::{ DenseSamplesRef, FieldSpec, HistogramSamples, SampleRef, Samples, Series, SeriesFields, SeriesSchema, @@ -301,130 +299,6 @@ fn merge_keeps_histograms_and_rejects_any_doubled_field() { } } -/// The histogram payloads of one label set merge by position, whatever table each shard filed its -/// own payload in. -/// -/// A label set reaches a merge as one fragment per producer, and every fragment was built by its own -/// builder: each holds its own histogram table and its own indices into it, so the merged field has -/// to be rebuilt over the union of the positions with each row still naming the payload it was -/// pushed with. The fragments here are complementary, the position neither of two fragments holds -/// stays a hole, and a fragment that pushes nothing adds no row. -#[test] -fn merge_keeps_each_shards_histogram_payload() { - // One histogram per shard, each with its own count, sum and bucket payload: `count/sum` are the - // two fields a reader of the merged series checks a payload by. - let payloads = [ - OwnedHistogram::parse( - "0/0/0/3ff0000000000000/3ff0000000000000/1/0_2/4000000000000000,4008000000000000///", - ), - OwnedHistogram::parse("0/0/0/4008000000000000/4014000000000000/1/1_1/4010000000000000///"), - OwnedHistogram::parse("0/0/0/4014000000000000/4018000000000000/1/0_1/4014000000000000///"), - ]; - // The `(sum, count)` each payload was written with, in payload order. - let scalars = [(1.0, 1.0), (5.0, 3.0), (6.0, 5.0)]; - let schema = grid_schema(1, 8); - let shard = |rows: &[(usize, usize)]| { - let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap(); - for &(point, payload) in rows { - let histogram = FloatHistogram::try_from_view(&payloads[payload].view()).unwrap(); - builder - .push_samples(point, &[Some(GridSample::Histogram(Box::new(histogram)))]) - .unwrap(); - } - builder - .finish(metric(0)) - .unwrap() - .expect("the shard holds the rows it pushed") - }; - // A shard of the same label set holding nothing finishes with no series at all, so it adds - // neither a row nor a gap to what the others pushed. - assert!( - GridSeriesBuilder::try_new(schema.clone()) - .unwrap() - .finish(metric(0)) - .unwrap() - .is_none(), - "an empty shard handed out a series" - ); - let merged = merge_same_labelset( - &schema, - vec![shard(&[(0, 0), (3, 1)]), shard(&[(1, 1)]), shard(&[(4, 2)])], - ) - .unwrap(); - assert_eq!(merged.len(), 1, "one label set, so one merged series"); - let field = merged[0].fields.get(0).expect("one field"); - let merged_rows = merged[0] - .time - .positions() - .map(|(point, row)| { - let SampleRef::Histogram(view) = field.sample(row) else { - panic!("position {point} of the merged series is not a histogram"); - }; - let histogram = FloatHistogram::try_from_view(&view).unwrap(); - (point, histogram.sum, histogram.count) - }) - .collect::>(); - // One row per valued position, in order: position 2 is a hole in every shard and stays a hole, - // and each row carries the payload its own shard pushed. - assert_eq!( - vec![ - (0, scalars[0].0, scalars[0].1), - (1, scalars[1].0, scalars[1].1), - (3, scalars[1].0, scalars[1].1), - (4, scalars[2].0, scalars[2].1), - ], - merged_rows - ); - // The merged field is one freshly built table over those rows: the rows the shards held are - // addressed by the merged series' own indices, so a row cannot name a shard's table. - let SampleRef::Histogram(first) = field.sample(0) else { - panic!("the merged field holds no histogram"); - }; - assert_eq!(&[2.0, 3.0], first.positive.buckets); - let SampleRef::Histogram(second) = field.sample(1) else { - panic!("the merged field holds no histogram"); - }; - assert_eq!(&[4.0], second.positive.buckets); - assert_eq!(1, second.positive.span_lengths.len()); - assert_eq!(1, second.positive.span_offsets[0]); -} - -/// Two shards of one label set holding a histogram at one position are refused. -/// -/// The merge collects the pieces of one label set into one series, so a position two pieces fill -/// would carry two values of one field: that is the collision Prometheus reports rather than a -/// choice for this node to make. The refusal names the shared label set, as it does for floats. -#[test] -fn merge_rejects_a_histogram_position_two_shards_hold() { - let payloads = [ - OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/1/0_1/4000000000000000///"), - OwnedHistogram::parse("0/0/0/4000000000000000/4000000000000000/1/0_1/4000000000000000///"), - ]; - let schema = grid_schema(1, 8); - let shard = |point: usize, payload: usize| { - let histogram = FloatHistogram::try_from_view(&payloads[payload].view()).unwrap(); - let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap(); - builder - .push_samples(point, &[Some(GridSample::Histogram(Box::new(histogram)))]) - .unwrap(); - builder.finish(metric(0)).unwrap().unwrap() - }; - // Complementary positions merge whatever payloads they hold, so the only reason the second pair - // is refused is the position both of them fill. - let merged = merge_same_labelset(&schema, vec![shard(0, 0), shard(1, 1)]).unwrap(); - assert_eq!(1, merged.len()); - assert_eq!( - 2, - merged[0].time.len(), - "two shards holding different positions merge" - ); - let error = merge_same_labelset(&schema, vec![shard(1, 0), shard(1, 1)]).unwrap_err(); - assert!( - error.to_string().contains("same labelset"), - "the refusal does not name the shared label set: {error}" - ); -} - /// Two values a million positions apart are one series with two rows, addressed by two runs: the /// positions between them cost neither a row nor a value buffer sized by the span. #[test] @@ -1587,122 +1461,3 @@ mod acceptance { assert_eq!(read(&three_chunks), read(&two_chunks)); } } - -/// The payload of one histogram, field by field, with every float compared by its bits so that a -/// value that moved by one bit is not equal. -fn payload(view: &HistogramView<'_>) -> Vec { - let hint = match view.reset_hint { - CounterResetHint::Unknown => 0, - CounterResetHint::CounterReset => 1, - CounterResetHint::NotCounterReset => 2, - CounterResetHint::Gauge => 3, - CounterResetHint::Unrecognized(value) => (value as u64) | (1 << 32), - }; - let mut bits = vec![ - view.schema as u64, - hint, - view.start_timestamp.map_or(u64::MAX, |start| start as u64), - view.zero_threshold.to_bits(), - view.zero_count.to_bits(), - view.count.to_bits(), - view.sum.to_bits(), - view.custom_values.len() as u64, - ]; - bits.extend(view.custom_values.iter().map(|value| value.to_bits())); - for side in [&view.positive, &view.negative] { - bits.push(side.span_offsets.len() as u64); - bits.push(side.span_lengths.len() as u64); - bits.push(side.buckets.len() as u64); - bits.extend(side.span_offsets.iter().map(|&offset| offset as u64)); - bits.extend(side.span_lengths.iter().map(|&length| length as u64)); - bits.extend(side.buckets.iter().map(|bucket| bucket.to_bits())); - } - bits -} - -/// A value pushed from the view it was read as lands in the result as the payload the owned route -/// writes, start timestamp included: a selection that only hands a sample on leaves it as it is. -#[test] -fn a_pushed_view_writes_the_payload_of_the_owned_route() { - let schema = grid_schema(2, 3); - let histograms = [ - OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///"), - OwnedHistogram::parse("0/0/0/4000000000000000/4008000000000000/1/0_1/4000000000000000///"), - ]; - // The sample a scan read: the payload of its row plus the start timestamp the row carried. - let mut stamped = histograms[1].view(); - stamped.start_timestamp = Some(1_500); - let views = [histograms[0].view(), stamped]; - - let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap(); - builder - .push_samples( - 0, - &[ - Some(GridSample::Histogram(Box::new( - FloatHistogram::try_from_view(&views[0]).unwrap(), - ))), - Some(GridSample::Float(2.0)), - ], - ) - .unwrap(); - builder - .push_samples( - 2, - &[ - None, - Some(GridSample::Histogram(Box::new( - FloatHistogram::try_from_view(&views[1]).unwrap(), - ))), - ], - ) - .unwrap(); - let owned = builder.finish(metric(0)).unwrap().unwrap(); - - let mut builder = GridSeriesBuilder::try_new(schema).unwrap(); - builder - .push_refs(0, &[SampleRef::Histogram(views[0]), SampleRef::Float(2.0)]) - .unwrap(); - builder - .push_refs(2, &[SampleRef::Missing, SampleRef::Histogram(views[1])]) - .unwrap(); - let by_view = builder.finish(metric(0)).unwrap().unwrap(); - - let described = |series: &Series, field: usize| -> Vec<(usize, String, Option>)> { - let samples = &series.fields.as_slice()[field]; - series - .time - .positions() - .map(|(point, row)| match samples.sample(row) { - SampleRef::Missing => (point, "missing".to_string(), None), - SampleRef::Float(value) => { - (point, "float".to_string(), Some(vec![value.to_bits()])) - } - SampleRef::Histogram(view) => { - (point, "histogram".to_string(), Some(payload(&view))) - } - }) - .collect() - }; - for field in 0..2 { - assert_eq!( - described(&owned, field), - described(&by_view, field), - "field {field} came back different through the view route" - ); - } - assert_eq!( - described(&by_view, 1), - vec![ - (0, "float".to_string(), Some(vec![2.0f64.to_bits()])), - (2, "histogram".to_string(), Some(payload(&views[1]))), - ], - "the pushed view lost the payload it was read with" - ); - // The comparison covers the start timestamp, which the two routes had to preserve. - assert_ne!( - payload(&views[1]), - payload(&histograms[1].view()), - "the start timestamp is part of what is compared" - ); -} diff --git a/src/promql/src/histogram_eval/aggregate.rs b/src/promql/src/histogram_eval/aggregate.rs index 2c730d7e43a..5583be6ecec 100644 --- a/src/promql/src/histogram_eval/aggregate.rs +++ b/src/promql/src/histogram_eval/aggregate.rs @@ -15,7 +15,6 @@ //! `sum` and `avg` of native histograms within one aggregation group, following the histogram //! paths of Prometheus `aggregation`. -use std::borrow::Cow; use std::sync::Arc; use datafusion::arrow::array::UInt32Array; @@ -140,7 +139,7 @@ fn aggregate_point<'a>( has_float = true; None } - SampleRef::Histogram(view) => Some(FloatHistogram::try_from_view(&view).map(Cow::Owned)), + SampleRef::Histogram(view) => Some(FloatHistogram::try_from_view(&view)), }); let (value, _) = sum_or_avg(op, histograms.by_ref())?; // A fold stopped by mixed bucket kinds still reports conversion errors of later samples. @@ -160,17 +159,12 @@ fn aggregate_point<'a>( /// time, so the first error ends the fold. The result is derived, so it carries no start /// timestamp. /// -/// The histograms are carried by reference: the accumulator is the only value the fold copies -/// out, and every sample after the first is only read. A window of `w` samples therefore costs -/// one deep copy, not `w`, and a caller that already owns its window pays none for the samples it -/// keeps. -/// /// The flag reports whether any add had to reconcile two different custom-bucket sets, which is /// what Prometheus annotates with an info. It is carried out of the fold rather than recomputed: /// the fold already calls [`FloatHistogram::kahan_add`] and receives that outcome. -pub(crate) fn sum_or_avg<'a>( +pub(crate) fn sum_or_avg( op: HistogramAggregate, - histograms: impl IntoIterator>>, + histograms: impl IntoIterator>, ) -> Result<(Option, bool)> { let mut value: Option = None; let mut compensation: Option = None; @@ -181,25 +175,23 @@ pub(crate) fn sum_or_avg<'a>( let h = h?; count += 1.0; let Some(current) = value.as_mut() else { - // The one copy the fold takes: the accumulator starts as the first sample. - value = Some(h.into_owned()); + value = Some(h); continue; }; - let h = h.as_ref(); // Go drops the result with a warning for this addition error only; others propagate. if current.uses_custom_buckets() != h.uses_custom_buckets() { return Ok((None, custom_reconciled)); } match op { HistogramAggregate::Sum => { - let (c, outcome) = current.kahan_add(h, compensation.take())?; + let (c, outcome) = current.kahan_add(&h, compensation.take())?; custom_reconciled |= outcome.custom_bounds_reconciled; compensation = Some(c); } HistogramAggregate::Avg => { if mean.is_none() { let mut sum = current.clone(); - let (c, outcome) = sum.kahan_add(h, compensation.clone())?; + let (c, outcome) = sum.kahan_add(&h, compensation.clone())?; custom_reconciled |= outcome.custom_bounds_reconciled; if !sum.has_overflow() { *current = sum; @@ -218,9 +210,7 @@ pub(crate) fn sum_or_avg<'a>( if let Some(c) = compensation.as_mut() { c.mul(q); } - // The incremental mean scales the sample it adds, so this is the one branch that - // needs the histogram by value on the sample after the first. - let mut to_add = h.clone(); + let mut to_add = h; to_add.div(count); let mean = mean.as_mut().expect("set above"); mean.mul(q); diff --git a/src/promql/src/histogram_eval/float_histogram.rs b/src/promql/src/histogram_eval/float_histogram.rs index dd13215fae7..1dd0b6e87f5 100644 --- a/src/promql/src/histogram_eval/float_histogram.rs +++ b/src/promql/src/histogram_eval/float_histogram.rs @@ -31,46 +31,8 @@ use crate::histogram_eval::native::{CUSTOM_BUCKETS_SCHEMA, bound}; type Buckets = Vec<(i32, f64)>; -/// How often a histogram of this thread was copied or built from a view, so a test can hold a path -/// to the copies it should not make rather than to the shape of its arguments. Test-only: a copy -/// and a borrow produce the same value, so a test comparing only values could not tell the two -/// apart. Both counters belong to the calling thread, so tests running side by side cannot see -/// each other's work. -#[cfg(test)] -pub(crate) mod copies { - use std::cell::Cell; - - thread_local! { - /// Deep copies [`super::FloatHistogram::clone`] made on this thread. - static CLONES: Cell = const { Cell::new(0) }; - /// Owned histograms [`super::FloatHistogram::try_from_view`] built on this thread. - static MATERIALIZED: Cell = const { Cell::new(0) }; - } - - /// Records one deep copy. - pub(crate) fn record_clone() { - CLONES.with(|clones| clones.set(clones.get() + 1)); - } - - /// Records one owned histogram built from a view. - pub(crate) fn record_materialized() { - MATERIALIZED.with(|built| built.set(built.get() + 1)); - } - - /// The deep copies made on this thread since the last call, which starts a new count. - pub(crate) fn take_clones() -> usize { - CLONES.with(|clones| clones.replace(0)) - } - - /// The owned histograms built from a view on this thread since the last call, which starts a - /// new count. - pub(crate) fn take_materialized() -> usize { - MATERIALIZED.with(|built| built.replace(0)) - } -} - /// A native histogram with absolute bucket counts. -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct FloatHistogram { pub(crate) reset_hint: CounterResetHint, /// The start timestamp of the sample this histogram was read from, in milliseconds. @@ -112,30 +74,8 @@ pub(crate) struct AddOutcome { pub(crate) custom_bounds_reconciled: bool, } -impl Clone for FloatHistogram { - fn clone(&self) -> Self { - #[cfg(test)] - copies::record_clone(); - Self { - reset_hint: self.reset_hint, - start_timestamp: self.start_timestamp, - schema: self.schema, - zero_threshold: self.zero_threshold, - zero_count: self.zero_count, - count: self.count, - sum: self.sum, - positive: self.positive.clone(), - negative: self.negative.clone(), - custom_values: self.custom_values.clone(), - layout: self.layout.clone(), - } - } -} - impl FloatHistogram { pub(crate) fn try_from_view(view: &HistogramView<'_>) -> Result { - #[cfg(test)] - copies::record_materialized(); Ok(Self { reset_hint: view.reset_hint, start_timestamp: view.start_timestamp, diff --git a/src/promql/src/histogram_eval/tests.rs b/src/promql/src/histogram_eval/tests.rs index 3c1834b4e3e..12e783b68f7 100644 --- a/src/promql/src/histogram_eval/tests.rs +++ b/src/promql/src/histogram_eval/tests.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::borrow::Cow; use std::collections::BTreeMap; use std::sync::Arc; @@ -23,17 +22,16 @@ use datatypes::series::histogram::{ use datatypes::series::samples::{FieldSpec, SampleRef, Series, SeriesSchema}; use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis}; -use super::float_histogram::copies; use super::float_histogram::{AddOutcome, FloatHistogram}; use super::{ HistogramAggregate, HistogramFunction, HistogramParameters, NativeHistogramFunction, aggregate_histograms, evaluate_histograms, evaluate_histograms_with_annotations, - histogram_quantiles, histogram_string, map_native_histogram, sum_or_avg, + histogram_quantiles, histogram_string, map_native_histogram, }; use crate::grid_series::merge_same_labelset; use crate::grid_series::reference::{ - GoldenCase, Labels, OwnedHistogram, Points, assert_histogram_close, canonical_bits, close, - golden_cases, labels_of, metric_of, mixed_series, points_by_labels, points_of, + GoldenCase, Labels, OwnedHistogram, Points, canonical_bits, close, golden_cases, labels_of, + metric_of, mixed_series, points_by_labels, points_of, }; fn schema(case: &GoldenCase) -> Arc { @@ -1067,81 +1065,3 @@ fn classic_histograms_keep_metadata_identity_until_after_evaluation() { ); } } - -/// A window of native histograms is folded by reference: the sum copies the accumulator it starts -/// from and nothing else, so four samples cost one deep copy rather than four. The fold itself is -/// the one the owned samples go through, bit for bit. -#[test] -fn a_histogram_sum_copies_one_sample_of_a_window() { - let window: Vec = [ - "0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///", - "0/0/0/4000000000000000/4008000000000000/0/1_1/4000000000000000///", - "0/0/0/4008000000000000/4010000000000000/0/1_1/4008000000000000///", - "0/0/0/4010000000000000/4014000000000000/0/1_1/4010000000000000///", - ] - .iter() - .map(|text| FloatHistogram::try_from_view(&OwnedHistogram::parse(text).view()).unwrap()) - .collect(); - - copies::take_clones(); - let (sum, reconciled) = sum_or_avg( - HistogramAggregate::Sum, - window.iter().map(|h| Ok(Cow::Borrowed(h))), - ) - .unwrap(); - assert_eq!( - copies::take_clones(), - 1, - "the sum copied every sample of the window, not only the accumulator it starts from" - ); - assert!(!reconciled); - - // The same window as owned samples, taken before the count: the accumulator arrives by value, - // so the fold copies nothing and must produce the result the borrowed fold produced. - let owned_samples: Vec> = - window.iter().cloned().map(Cow::Owned).collect(); - copies::take_clones(); - let (owned, _) = - sum_or_avg(HistogramAggregate::Sum, owned_samples.into_iter().map(Ok)).unwrap(); - assert_eq!(copies::take_clones(), 0, "an owned sample was copied"); - - let sum = sum.expect("the window sums to a histogram"); - assert_histogram_close( - &sum, - &owned.expect("the window sums to a histogram"), - 0.0, - "the borrowed sum", - ); -} - -/// The mean reads the window by reference too, and takes the same samples by value as the owned -/// fold does: the sum `kahan_add` keeps is copied either way, and the values agree bit for bit. -#[test] -fn a_histogram_mean_reads_its_window_by_reference() { - let window: Vec = [ - "0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///", - "0/0/0/4000000000000000/4008000000000000/0/1_1/4000000000000000///", - "0/0/0/4008000000000000/4010000000000000/0/1_1/4008000000000000///", - ] - .iter() - .map(|text| FloatHistogram::try_from_view(&OwnedHistogram::parse(text).view()).unwrap()) - .collect(); - - let borrowed = sum_or_avg( - HistogramAggregate::Avg, - window.iter().map(|h| Ok(Cow::Borrowed(h))), - ) - .unwrap(); - let owned = sum_or_avg( - HistogramAggregate::Avg, - window.iter().cloned().map(|h| Ok(Cow::Owned(h))), - ) - .unwrap(); - assert_eq!(borrowed.1, owned.1); - assert_histogram_close( - &borrowed.0.expect("the window averages to a histogram"), - &owned.0.expect("the window averages to a histogram"), - 0.0, - "the borrowed mean", - ); -} diff --git a/src/promql/src/instant_eval.rs b/src/promql/src/instant_eval.rs index 08e9ce51d3a..0b8b092bf4c 100644 --- a/src/promql/src/instant_eval.rs +++ b/src/promql/src/instant_eval.rs @@ -27,22 +27,6 @@ use crate::map::drop_metric_name; #[cfg(test)] mod tests; -/// The source rows [`select_instants`] examined, counted per row. -/// -/// The selector advances one cursor over the axis, so the rows it reads can never outnumber the -/// rows of the axis: one pass, not one per grid point. Counting them is what keeps a second pass -/// over the axis from being added back unobserved, since the selection it returns is the same -/// either way. -#[cfg(test)] -pub(crate) static AXIS_ROWS_READ: std::sync::atomic::AtomicU64 = - std::sync::atomic::AtomicU64::new(0); - -/// Counts one row of the axis examined; a no-op outside tests. -fn row_read() { - #[cfg(test)] - AXIS_ROWS_READ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); -} - /// Selects the one source row an anchored instant reads, with the lookback, offset and staleness /// rules of [`select_instants`]: the last sample at or before `at_ms` (shifted by `offset_ms`) and /// not more than the lookback before it, unless a stale marker is the latest sample there. @@ -117,13 +101,10 @@ pub(crate) fn instant_candidate_span( /// Selects source rows for an aligned PromQL instant-evaluation range. /// -/// Returns original logical row indices and evaluation timestamps from a complete source axis of -/// strictly increasing milliseconds. Any provided field's stale marker suppresses selection; +/// Returns original logical row indices and evaluation timestamps from a complete source axis +/// of strictly increasing milliseconds. Any provided field's stale marker suppresses selection; /// missing samples are not stale and remain selectable. /// -/// The ordering of the axis is [`RawTimestamps`]' own guarantee, checked when it was built, so this -/// walks the axis once -- one cursor advance per row -- and validates nothing about it again. -/// /// The two vectors it returns hold one `u64` source row index and one `i64` evaluation timestamp /// per selected point, so what it allocates is 16 bytes per point of the clipped grid rather than /// anything proportional to the samples of the axis. It reserves none of that: every caller grows @@ -197,7 +178,6 @@ pub fn select_instants( }; current_stale |= stale; } - row_read(); cursor += 1; } let Some(candidate) = cursor.checked_sub(1) else { diff --git a/src/promql/src/instant_eval/tests.rs b/src/promql/src/instant_eval/tests.rs index 036c948a53a..cd2b3926426 100644 --- a/src/promql/src/instant_eval/tests.rs +++ b/src/promql/src/instant_eval/tests.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use std::ops::RangeInclusive; use std::sync::Arc; -use std::sync::atomic::Ordering; use datatypes::arrow::array::{Float64Array, UInt32Array}; use datatypes::arrow::buffer::ScalarBuffer; @@ -29,8 +28,8 @@ use datatypes::series::samples::{ use datatypes::series::time::{EvalGrid, GridSpec}; use super::{ - AXIS_ROWS_READ, InstantCandidateSpan, info_sample_times, instant_candidate_span, - select_instants, timestamp_of_evaluated, timestamp_of_selection, + InstantCandidateSpan, info_sample_times, instant_candidate_span, select_instants, + timestamp_of_evaluated, timestamp_of_selection, }; use crate::grid_series::GridSeriesBuilder; use crate::grid_series::reference::{Labels, labels_of, metric_of}; @@ -234,42 +233,6 @@ fn offset_extreme_timestamps_are_checked_without_large_fixture() { assert_eq!(times, [1_000_000]); } -/// A selection walks its axis once, whatever the grid it selects onto. -/// -/// The strictly increasing order of the timestamps is `RawTimestamps`' guarantee, checked when the -/// axis is built, so the selector has nothing left to validate: the rows it reads are the rows of -/// the axis, once each, and a second pass over them -- such as a re-check of their order -- would -/// show up here as twice as many. The selection itself is the same either way, which is why the -/// count has to be what the test reads. -#[test] -fn a_selection_reads_its_axis_once() { - let rows = 5_000; - let times: Vec = (0..rows).map(|row| row as i64).collect(); - let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0; rows]))); - let read = |range: RangeInclusive, step: i64| -> u64 { - AXIS_ROWS_READ.store(0, Ordering::Relaxed); - let (selected, _) = select(×, &fields, range, step, 10, 0).unwrap(); - assert!(!selected.is_empty(), "the selection is what reads the axis"); - AXIS_ROWS_READ.load(Ordering::Relaxed) - }; - // Every row is selected: one pass, no more. - assert_eq!( - read(0..=rows as i64 - 1, 1), - rows as u64, - "the axis was walked more than once" - ); - // Fewer points over the whole axis read it once as well: the pass follows the samples, not the - // points, so it never exceeds the rows of the axis however many points the grid holds. - for step in [10, 499] { - let visited = read(0..=rows as i64 - 1, step); - assert!( - visited <= rows as u64, - "step {step} read {visited} rows of a {} row axis", - rows - ); - } -} - #[test] fn instant_candidate_span_preserves_unaligned_and_wide_ranges() { // The query end is not on the start's lattice. The scan bound is 125, but stepping from 100 diff --git a/src/promql/src/native/canonicalize.rs b/src/promql/src/native/canonicalize.rs index fefe2d88495..900b85d09c5 100644 --- a/src/promql/src/native/canonicalize.rs +++ b/src/promql/src/native/canonicalize.rs @@ -23,10 +23,10 @@ //! //! # What decides it //! -//! Only a rewrite can alias, and only one that writes a label its input already carries or drops one -//! its output would still be told apart by: [`duplicates_canonical_labels`] is that fact, derived -//! from the tree and the tags each source reports. A tree with no such rewrite gains no merge at -//! all, so a per-series query keeps every scan partition and pays nothing for this pass. +//! Only a rewrite can alias, and only one that writes a label its input already carries: +//! [`duplicates_canonical_labels`] is that fact, derived from the tree and the tags each source +//! reports. A tree with no such rewrite gains no merge at all, so a per-series query keeps every +//! scan partition and pays nothing for this pass. //! //! Where a merge is added is a property of the consumer, and of what the operations between the //! consumer and the rewrite do to one sample: @@ -45,39 +45,6 @@ //! A merge is never added below a raw scan, a scalar or aggregate state: it binds samples evaluated //! on a grid, and no consumer asks for one over those. //! -//! A node can be a source of duplicates without rewriting a label its input carries: a range -//! function that computes its result and `timestamp` in both of its forms drop the metadata labels - -//! the metric name, `__type__` and `__unit__` - and hand out a set those no longer tell apart. What -//! they produce is collected by their consumer - the root included - exactly as a rewrite of an -//! existing label is, unless -//! [`duplicates_canonical_labels`](crate::native::plan::duplicates_canonical_labels) proves that -//! nothing they drop was telling two of their series apart: a source bound to one metric names -//! every series it hands over, a source whose tags carry neither `__type__` nor `__unit__` holds no -//! series those two tell apart either, and the scan's key columns always decode a value. -//! `rate(some_metric[5m])` over such a scan is then a window that streams its scan, and the same -//! query over a selector that discovered several metrics keeps the collection it needs. The same -//! rule - the labels and the scan's keys - is what a math or histogram map, a parameterized map and -//! an arithmetic or `bool` scalar broadcast are judged by. -//! -//! The key half is not the tree's to answer, so the caller states it as `keys_survive`: two stored -//! series of one metric whose key column holds `NULL` in one and `''` in the other decode to an -//! absent and to an empty label, and the drop removes the empty one with the name, leaving both -//! under one label set. The rule is -//! [`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), -//! which the physical per-series scan reads to decide whether a node may stream and the planner -//! reads from the resolved table, so the two cannot disagree about a drop. A caller that cannot -//! state it passes `false`; the statements collected for that reason alone are counted -//! ([`leaves_a_drop_to_the_storage_keys`](crate::native::plan::leaves_a_drop_to_the_storage_keys)). -//! -//! A drop above a merge is not asked about the empty value it removes, which is what keeps a chain of -//! drops to one collection: [`NativeExpr::Merge`] hands over one series per canonical label set - the -//! comparison that treats an empty label value as no label - so no drop above it can fold two of its -//! series by dropping an empty one. The metadata labels a drop removes are another matter: a merge -//! keeps two series that differ in `__name__`, `__type__` or `__unit__`, so the labels half of the -//! proof is still asked above it. With that, `abs(abs(rate(x[5m])))` over a table whose key columns -//! can spell the `NULL`/`''` pair pays for the one merge at the bottom of the chain rather than one -//! per drop. -//! //! Canonicalizing a map's input is what makes the histogram case work: in //! `histogram_count(label_replace(x, "host", "same", "host", ".*"))` two scan partitions each //! rewrite one series to `{host="same"}` and `histogram_count` drops both - they are float samples - @@ -94,8 +61,6 @@ //! execution is the lowering's question. A merge that spans partitions uses a producer and the //! existing canonical router; downstream consumers read ordinary evaluated frames. -use std::sync::Arc; - use datafusion::error::Result as DfResult; use crate::native::plan::{ @@ -112,20 +77,6 @@ mod tests; /// source the caller says nothing about proves nothing, and a rewrite over it is collected. The /// index is what a leaf reads, so a tree with several sources is judged source by source. /// -/// `source_metrics` is the other half of that knowledge, indexed the same way: the one metric each -/// source is bound to, or `None` when it may hand over several. It is what lets a kernel that drops -/// the metadata labels keep every partition - see -/// [`duplicates_canonical_labels`](crate::native::plan::duplicates_canonical_labels) - and a caller -/// that cannot say collects instead. -/// -/// `keys_survive` is the last fact and the one the tree cannot carry: whether every scan below -/// `tree` still tells its series apart once a drop has normalized their labels -/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), -/// read from the resolved table). A key column that holds `NULL` in one series and `''` in another -/// decodes to an absent and to an empty label, and the empty one leaves with the metadata labels, so -/// `false` collects a drop the labels alone would have proved. A caller that cannot state it passes -/// `false`. -/// /// Every operation binding and every `source_index` stays where it was, so the rewrite changes which /// series a consumer reads and not what the query computes. The result is checked by /// [`NativeTree::try_new`], because a merge has to sit over an evaluated vector and the pass may @@ -133,12 +84,10 @@ mod tests; pub fn merge_label_rewrites( tree: &NativeTree, source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, ) -> DfResult { - let root = rewrite(tree.root(), source_tags, source_metrics, keys_survive)?; + let root = rewrite(tree.root(), source_tags)?; // The root is a consumer too: whatever reads this tree reads one series per label set. - NativeTree::try_new(canonical(root, source_tags, source_metrics, keys_survive)) + NativeTree::try_new(canonical(root, source_tags)) } /// Wraps `node` in a [`NativeExpr::Merge`] when its pieces can share a label set. @@ -151,19 +100,14 @@ pub fn merge_label_rewrites( /// ([`duplicates_canonical_labels`]), because collecting pieces that are already distinct is a /// barrier that buys nothing; /// - it is not already a merge, which would collect what is already collected. -fn canonical( - node: NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> NativeNode { +fn canonical(node: NativeNode, source_tags: &[Vec]) -> NativeNode { if !matches!(&node.output, NativeType::Samples(schema) if schema.eval_grid.is_some()) { return node; } if matches!(node.expr, NativeExpr::Merge { .. }) { return node; } - if !duplicates_canonical_labels(&node, source_tags, source_metrics, keys_survive) { + if !duplicates_canonical_labels(&node, source_tags) { return node; } // A merge reports its input's layout: the same fields on the same grid. @@ -178,12 +122,7 @@ fn canonical( /// Rewrites `node` bottom-up: its children first, then the merge its own consumer contract needs of /// them. -fn rewrite( - node: &NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> DfResult { +fn rewrite(node: &NativeNode, source_tags: &[Vec]) -> DfResult { let expr = match &node.expr { // A source is the boundary it was bound to and there is nothing below it to rewrite. A // constant reads nothing. @@ -194,27 +133,22 @@ fn rewrite( // grid; a timestamp replaces each value with its sample's time. Both read each piece as it // is, so neither collects what it reads. NativeExpr::Instant { input, op } => NativeExpr::Instant { - input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), + input: Box::new(rewrite(input, source_tags)?), op: op.clone(), }, NativeExpr::Timestamp { input } => NativeExpr::Timestamp { - input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), + input: Box::new(rewrite(input, source_tags)?), }, // A merge is the canonicalizer itself, so it asks nothing of its input: fragments are what it // was written to receive. NativeExpr::Merge { input } => NativeExpr::Merge { - input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), + input: Box::new(rewrite(input, source_tags)?), }, // A window folds a range of one series' samples and hands the answer over as that series, so // it reads the series whole. Over a raw scan the pieces are whole storage series already and // the merge above is a no-op; over an evaluated vector a rewrite can have aliased them. NativeExpr::Window { input, op } => NativeExpr::Window { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: op.clone(), }, // A map is point-wise, but a point can drop out of its output: the histogram functions drop @@ -223,12 +157,7 @@ fn rewrite( // merge above this map would be handed an occupancy the map may already have erased - the // pieces have to be collected where they still hold their samples, below it. NativeExpr::Map { input, op } => NativeExpr::Map { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: op.clone(), }, // A parameterized map's parameters are scalars, rewritten but never collected - a scalar @@ -239,38 +168,23 @@ fn rewrite( op, parameters, } => NativeExpr::ParameterizedMap { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: *op, parameters: parameters .iter() - .map(|parameter| rewrite(parameter, source_tags, source_metrics, keys_survive)) + .map(|parameter| rewrite(parameter, source_tags)) .collect::>>()?, }, // A fold answers its groups, so two pieces of one series in a group are summed as if the // query had written them twice: the merge is what makes one series arrive as one. NativeExpr::Aggregate { input, op } => NativeExpr::Aggregate { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: op.clone(), }, // Vector operations consume canonical series: ranking counts them, and presence checks and // folds must not silently consume overlapping fragments that a merge would reject. NativeExpr::Vector { input, op } => NativeExpr::Vector { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: op.clone(), }, // Each side of a binary or set operation is matched by label on its own, so each is @@ -281,33 +195,13 @@ fn rewrite( right, matching, } => NativeExpr::Binary { - left: Box::new(canonical( - rewrite(left, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), - right: Box::new(canonical( - rewrite(right, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + left: Box::new(canonical(rewrite(left, source_tags)?, source_tags)), + right: Box::new(canonical(rewrite(right, source_tags)?, source_tags)), matching: matching.clone(), }, NativeExpr::Set { left, right, op } => NativeExpr::Set { - left: Box::new(canonical( - rewrite(left, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), - right: Box::new(canonical( - rewrite(right, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + left: Box::new(canonical(rewrite(left, source_tags)?, source_tags)), + right: Box::new(canonical(rewrite(right, source_tags)?, source_tags)), op: op.clone(), }, // `info` pairs each base series with the info series that share its identifying labels, so it @@ -318,33 +212,18 @@ fn rewrite( info, selector, } => NativeExpr::Info { - base: Box::new(canonical( - rewrite(base, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), - info: Box::new(canonical( - rewrite(info, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + base: Box::new(canonical(rewrite(base, source_tags)?, source_tags)), + info: Box::new(canonical(rewrite(info, source_tags)?, source_tags)), selector: selector.clone(), }, // `vector(s)` builds its series from a scalar, which carries no label set to collect. NativeExpr::VectorOf { input } => NativeExpr::VectorOf { - input: Box::new(rewrite(input, source_tags, source_metrics, keys_survive)?), + input: Box::new(rewrite(input, source_tags)?), }, // `scalar()` answers from the whole vector: a series that arrives in pieces would be seen // as several samples, so the pieces are collected where they still hold their samples. NativeExpr::ScalarOf { input } => NativeExpr::ScalarOf { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), }, // `topk` ranks the whole vector, so it counts canonical series; its parameter is a scalar, // rewritten but never collected - a scalar carries no label set. @@ -353,19 +232,9 @@ fn rewrite( op, parameter, } => NativeExpr::ParameterizedTopK { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: op.clone(), - parameter: Box::new(rewrite( - parameter, - source_tags, - source_metrics, - keys_survive, - )?), + parameter: Box::new(rewrite(parameter, source_tags)?), }, // A histogram function folds the bucket series of one histogram, so overlapping fragments // must not reach it; its parameters are scalars and only need rewriting. @@ -374,16 +243,11 @@ fn rewrite( op, parameters, } => NativeExpr::ParameterizedHistogram { - input: Box::new(canonical( - rewrite(input, source_tags, source_metrics, keys_survive)?, - source_tags, - source_metrics, - keys_survive, - )), + input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)), op: op.clone(), parameters: parameters .iter() - .map(|parameter| rewrite(parameter, source_tags, source_metrics, keys_survive)) + .map(|parameter| rewrite(parameter, source_tags)) .collect::>>()?, }, }; diff --git a/src/promql/src/native/canonicalize/tests.rs b/src/promql/src/native/canonicalize/tests.rs index 48ed4fda499..8b58bb8b21b 100644 --- a/src/promql/src/native/canonicalize/tests.rs +++ b/src/promql/src/native/canonicalize/tests.rs @@ -252,31 +252,6 @@ fn set_over(left: NativeNode, right: NativeNode, out: Arc) -> NativeNo } } -/// `last_over_time(inner[5m])`, a window that folds `inner` onto `out` and keeps the metric name. -fn keeping_window_over( - inner: NativeNode, - input_grid: Option>, - out: Arc, -) -> NativeNode { - NativeNode { - expr: NativeExpr::Window { - input: Box::new(inner), - op: BoundWindow { - function: RangeFunction::LastOverTime, - window: RangeWindow { - range_ms: 300_000, - offset_ms: 0, - anchor_ms: None, - modifier: None, - }, - input_grid, - output_grid: Arc::clone(&out), - }, - }, - output: evaluated(fields(1), out), - } -} - /// `rate(inner[5m])`, a window that folds `inner` onto `out` and drops the metric name. fn window_over( inner: NativeNode, @@ -353,54 +328,9 @@ fn merges(tree: &NativeTree) -> usize { } /// Runs the pass over `root`, whose sources report `source_tags`. -/// -/// Nothing is stated about which metric each source reads, so a kernel that drops the metadata -/// labels over one of them proves nothing and is collected. fn passed(root: NativeNode, source_tags: &[Vec]) -> NativeTree { - passed_over(root, source_tags, &[]) -} - -/// Runs the pass over `root`, whose sources report `source_tags` and read the metric each of -/// `source_metrics` names, indexed the way `merge_label_rewrites` takes it. -/// -/// The scans below these fixtures key their series by columns that always decode a value, which is -/// the case a metadata drop has to be proved against the labels alone: `NULL` and `''` in one key -/// column are what [`passed_over_nullable_keys`] states instead. -fn passed_over( - root: NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], -) -> NativeTree { - passed_over_with_keys(root, source_tags, source_metrics, true) -} - -/// The same pass over a tree whose scans key their series by a column that can hold `NULL` in one -/// series and `''` in another, so no metadata drop below it can be proved by the labels alone. -fn passed_over_nullable_keys( - root: NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], -) -> NativeTree { - passed_over_with_keys(root, source_tags, source_metrics, false) -} - -/// Runs the pass over `root` with the key fact the caller states, which is what -/// `merge_label_rewrites` takes as `keys_survive`. -fn passed_over_with_keys( - root: NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> NativeTree { let tree = NativeTree::try_new(root).expect("the fixture is a legal tree"); - merge_label_rewrites(&tree, source_tags, source_metrics, keys_survive) - .expect("the pass produces a legal tree") -} - -/// The labels a scan bound to one metric reports: its tag, and the name the binding gives every -/// series it hands over, which is not a tag column. -fn named_scan_tags() -> Vec> { - vec![tags(&["host", "__name__"])] + merge_label_rewrites(&tree, source_tags).expect("the pass produces a legal tree") } fn tags(names: &[&str]) -> Vec { @@ -508,7 +438,7 @@ fn presence_and_folding_vector_operations_collect_aliasing_inputs() { }; assert!(matches!(input.expr, NativeExpr::Map { .. }), "{op:?}"); - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); assert_eq!(1, merges(&twice), "{op:?}"); assert_eq!(tree.to_string(), twice.to_string(), "{op:?}"); } @@ -563,7 +493,7 @@ fn rewrites_after_ranking_or_grouping_are_collected() { ), "{name}" ); - let twice = merge_label_rewrites(&tree, &source_tags, &[], true).unwrap(); + let twice = merge_label_rewrites(&tree, &source_tags).unwrap(); assert_eq!(tree.to_string(), twice.to_string(), "{name}"); assert_eq!(1, merges(&twice), "{name}"); } @@ -574,8 +504,7 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() { let out = grid(0, 1000, 2); // A match answers one series per label set, but those labels are its operands': renaming `host` // to a constant above it puts both sides' results under one set, so the consumer above the - // rewrite is the one that has to collect them. The match itself reads its operands as they are; - // the one on the right drops the metadata labels, so its own answer is collected. + // rewrite is the one that has to collect them. The match itself reads its operands as they are. let matched = binary_over(selected(&out), abs_map(selected(&out)), Arc::clone(&out)); let renamed = map_ops( matched, @@ -584,11 +513,7 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() { ))], ); let tree = passed(renamed, &scan_tags()); - assert_eq!( - 2, - merges(&tree), - "the rewrite's own collection, and the one the name-dropping operand hands out: {tree}" - ); + assert_eq!(1, merges(&tree)); let NativeExpr::Merge { input } = &tree.root().expr else { panic!("the statement must collect the rewrite above the match"); }; @@ -599,9 +524,9 @@ fn a_rewrite_above_a_match_collects_the_matches_own_labels() { matches!(input.expr, NativeExpr::Binary { .. }), "the match is not handed the rewritten operand" ); - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); assert_eq!(tree.to_string(), twice.to_string()); - assert_eq!(2, merges(&twice)); + assert_eq!(1, merges(&twice)); } #[test] @@ -661,21 +586,18 @@ fn a_pointwise_map_reads_each_piece_as_it_is() { abs_map(rewrite(selected(&out), "host", "host")), &scan_tags(), ); - // Two collections: one below the map, because a map reads pieces point-wise and a point it - // reads can be one it drops, and one above it, for the metadata labels the map itself drops. - assert_eq!(2, merges(&tree), "{tree}"); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the map hands out: {tree}"); - }; - let NativeExpr::Map { input, .. } = &input.expr else { + assert_eq!(1, merges(&tree)); + // The map stays the root and its input is the merge: a map reads pieces point-wise, but a point + // it reads can be one it drops, so the collection has to happen below it. + let NativeExpr::Map { input, .. } = &tree.root().expr else { panic!("the root stays the point-wise map"); }; let NativeExpr::Merge { input } = &input.expr else { panic!("the map is handed one series per label set"); }; assert!(matches!(input.expr, NativeExpr::Map { .. })); - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); - assert_eq!(2, merges(&twice)); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + assert_eq!(1, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } @@ -691,15 +613,8 @@ fn a_sample_dropping_map_collects_below_itself() { ))], ); let tree = passed(histogram_count(aliasing), &scan_tags()); - assert_eq!( - 2, - merges(&tree), - "one merge below the sample-dropping map, one for the labels it drops" - ); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what `histogram_count` hands out: {tree}"); - }; - let NativeExpr::Map { input, .. } = &input.expr else { + assert_eq!(1, merges(&tree), "one merge, below the sample-dropping map"); + let NativeExpr::Map { input, .. } = &tree.root().expr else { panic!("the root stays `histogram_count`"); }; let NativeExpr::Merge { input } = &input.expr else { @@ -718,51 +633,36 @@ fn a_sample_dropping_map_collects_below_itself() { ); // A second pass over the collected tree changes nothing: the merge answers one series per label // set, so nothing below the map aliases any more. - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); - assert_eq!(2, merges(&twice)); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + assert_eq!(1, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } #[test] -fn a_pointwise_map_collects_its_own_name_drop() { +fn a_pointwise_map_over_a_clean_input_gains_no_merge() { let out = grid(0, 1000, 2); - // No rewrite anywhere in the tree, so no two pieces were told apart by a label one of them - // wrote - but `abs` drops the metadata labels, and the drop is a source of duplicates of its - // own: a key column that holds `NULL` in one stored series and `''` in another decodes to an - // absent and to an empty label, and both are gone once the empty one goes. Each map's answer is - // therefore collected, and both sit above the map that hands them out. + // No rewrite anywhere in the tree, so no two pieces can share a label set. let tree = passed(abs_map(abs_map(selected(&out))), &scan_tags()); - assert_eq!(2, merges(&tree), "{tree}"); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the outer map hands out: {tree}"); + assert_eq!(0, merges(&tree)); + let NativeExpr::Map { input, .. } = &tree.root().expr else { + panic!("the root stays the point-wise map"); }; - let NativeExpr::Map { input, .. } = &input.expr else { - panic!("the merge sits above the map, not below it: {tree}"); - }; - let NativeExpr::Merge { input } = &input.expr else { - panic!("the outer map reads one series per label set: {tree}"); - }; - assert!(matches!(input.expr, NativeExpr::Map { .. }), "{tree}"); - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); - assert_eq!(2, merges(&twice)); + assert!(matches!(input.expr, NativeExpr::Map { .. })); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); assert_eq!(tree.to_string(), twice.to_string()); } #[test] -fn a_sample_dropping_map_over_a_fresh_destination_collects_its_own_drop() { +fn a_sample_dropping_map_over_a_fresh_destination_gains_no_merge() { let out = grid(0, 1000, 2); // `dst` names no label the scan reports, so the rewrite cannot alias and the map's input needs - // no collection even though the map drops samples - but the metadata labels `histogram_count` - // drops are a source of duplicates of their own, so its answer is collected. + // no collection even though the map drops samples. let tree = passed( histogram_count(rewrite(selected(&out), "dst", "host")), &scan_tags(), ); - assert_eq!(1, merges(&tree), "a fresh destination cannot alias: {tree}"); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what `histogram_count` hands out: {tree}"); - }; - let NativeExpr::Map { input, .. } = &input.expr else { + assert_eq!(0, merges(&tree), "a fresh destination cannot alias"); + let NativeExpr::Map { input, .. } = &tree.root().expr else { panic!("the root stays `histogram_count`"); }; assert!(matches!(input.expr, NativeExpr::Map { .. })); @@ -780,15 +680,10 @@ fn a_parameterized_map_collects_below_itself() { ), &scan_tags(), ); - // One below the map, for the point a clamp can empty, and one above it, for the metadata labels - // the kernel drops. - assert_eq!(2, merges(&tree), "{tree}"); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the clamp hands out: {tree}"); - }; + assert_eq!(1, merges(&tree)); let NativeExpr::ParameterizedMap { input, parameters, .. - } = &input.expr + } = &tree.root().expr else { panic!("the root stays the parameterized map"); }; @@ -801,8 +696,8 @@ fn a_parameterized_map_collects_below_itself() { matches!(parameters[0].expr, NativeExpr::Scalar(_)), "a scalar argument is not collected" ); - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); - assert_eq!(2, merges(&twice)); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); + assert_eq!(1, merges(&twice)); assert_eq!(tree.to_string(), twice.to_string()); } @@ -814,16 +709,12 @@ fn each_side_of_a_match_is_collected_on_its_own() { binary_over(aliasing(), abs_map(selected(&out)), Arc::clone(&out)), &scan_tags(), ); - assert_eq!( - 2, - merges(&tree), - "the aliasing side and the name-dropping side are each collected: {tree}" - ); + assert_eq!(1, merges(&tree), "only the aliasing side needs collecting"); let NativeExpr::Binary { left, right, .. } = &tree.root().expr else { panic!("the root stays the match"); }; assert!(matches!(left.expr, NativeExpr::Merge { .. })); - assert!(matches!(right.expr, NativeExpr::Merge { .. })); + assert!(!matches!(right.expr, NativeExpr::Merge { .. })); } #[test] @@ -845,10 +736,9 @@ fn a_set_operation_collects_each_side() { #[test] fn a_window_collects_an_aliasing_child() { let out = grid(0, 1000, 2); - // A window over an evaluated vector folds one series' range, so it reads the series whole. The - // window keeps the metric name here, so the rewrite below it is the only source of duplicates. + // A window over an evaluated vector folds one series' range, so it reads the series whole. let tree = passed( - keeping_window_over( + window_over( rewrite(selected(&out), "host", "host"), Some(Arc::clone(&out)), Arc::clone(&out), @@ -866,614 +756,10 @@ fn a_window_collects_an_aliasing_child() { fn a_window_over_a_raw_scan_collects_nothing() { let out = grid(0, 1000, 2); // The scan hands over whole storage series, and a merge cannot take raw samples anyway. - let tree = passed(keeping_window_over(raw_input(0), None, out), &scan_tags()); + let tree = passed(window_over(raw_input(0), None, out), &scan_tags()); assert_eq!(0, merges(&tree)); } -#[test] -fn a_name_dropping_window_is_collected_by_its_consumer() { - let out = grid(0, 1000, 2); - // `rate` drops the metric name, which is part of the identity a merge groups by: a caller that - // says nothing about which metric the scan reads collects what the window hands out. - let tree = passed( - window_over(raw_input(0), None, Arc::clone(&out)), - &named_scan_tags(), - ); - assert_eq!(1, merges(&tree)); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the window hands out: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Window { .. }), - "the merge sits above the window, not below it" - ); -} - -#[test] -fn a_name_dropping_window_over_one_metric_streams() { - let out = grid(0, 1000, 2); - // The scan is bound to one metric, so every series it hands over is named the same thing: - // dropping that name tells none of them apart, and the window keeps every partition. - // - // The key columns are the other half of that proof, and this fixture states the half that holds: - // every one of them always decodes a value, so the empty label value the drop removes with the - // name never stood for a second stored series. - let tree = passed_over( - window_over(raw_input(0), None, Arc::clone(&out)), - &named_scan_tags(), - &[Some(Arc::from("some_metric"))], - ); - assert_eq!(0, merges(&tree), "{tree}"); - assert!( - matches!(tree.root().expr, NativeExpr::Window { .. }), - "the root stays the window: {tree}" - ); -} - -/// Two stored series of one metric that differ only in a tag column holding `NULL` in one and `''` -/// in the other: `{host="web-1", zone=""}` and `{host="web-1"}` are two series to the scan, and -/// `rate` drops the metric name and the empty `zone` with it, so both leave the window as -/// `{host="web-1"}`. -/// -/// The labels alone cannot tell them apart - they share host, zone is about to go, and the name is -/// the one every series of the scan shares - so the scan's key columns are what decides the answer. -/// A key column that can hold that pair is the fact that says so, and the drop is collected: the -/// merge is where the second value of one label set at one position is refused, which is -/// Prometheus's answer to two series of one label set rather than two pieces of one series. -/// -/// The execution half of this - that the merge refuses the pair instead of handing both over - is -/// `two_series_onto_one_label_set_after_a_window_are_rejected` in `native::execute::tests`. -#[test] -fn a_nullable_key_tag_collects_the_window_that_drops_an_empty_value() { - let out = grid(0, 1000, 2); - // A scan of one metric whose tags are `host` and `zone`, the second of which is a nullable key - // column: the pair above is one storage key away from being two. - let scan = [tags(&["host", "zone", "__name__"])]; - let single = [Some(Arc::::from("some_metric"))]; - let tree = passed_over_nullable_keys( - window_over(raw_input(0), None, Arc::clone(&out)), - &scan, - &single, - ); - assert!( - merges(&tree) >= 1, - "a nullable key column keeps the collection `rate` needs: {tree}" - ); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the window hands out: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Window { .. }), - "the merge sits above the window: {tree}" - ); - - // The same tree over a scan whose keys always decode a value: the labels prove the drop, and - // the window keeps every partition. - let tree = passed_over( - window_over(raw_input(0), None, Arc::clone(&out)), - &scan, - &single, - ); - assert_eq!( - 0, - merges(&tree), - "non-nullable keys keep the proof, so nothing is collected: {tree}" - ); -} - -#[test] -fn a_name_dropping_window_over_several_metrics_is_collected() { - let out = grid(0, 1000, 2); - // A selector that discovered several metrics binds no single name, so two of its series can be - // told apart by the name alone - and dropping it folds them into one label set. - let tree = passed_over( - window_over(raw_input(0), None, Arc::clone(&out)), - &named_scan_tags(), - &[None], - ); - assert_eq!(1, merges(&tree), "{tree}"); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "the root collects what the window hands out: {tree}" - ); -} - -#[test] -fn a_metadata_tag_in_the_binding_fails_closed() { - let out = grid(0, 1000, 2); - // `__type__` and `__unit__` go with the name, and a scan that carries one of them as a tag can - // hold two series that differ only there. A fixed metric name proves nothing about them, so the - // drop stays collected. - for metadata in ["__type__", "__unit__"] { - let scan = vec![tags(&["host", "__name__", metadata])]; - let tree = passed_over( - window_over(raw_input(0), None, Arc::clone(&out)), - &scan, - &[Some(Arc::from("some_metric"))], - ); - assert_eq!(1, merges(&tree), "{metadata}: {tree}"); - } -} - -/// The name a name-dropping kernel removes is the one every series of a single-metric scan shares, -/// and nothing else it drops tells two of them apart, so a math map, a histogram map, a -/// parameterized math kernel and an arithmetic scalar broadcast handed such a source keep every -/// partition. -/// -/// The scan's key columns are the other half of that proof, and this is the case they hold for: -/// every one of them always decodes a value, so the empty label value the drop removes with the name -/// never stood for a second stored series. [`a_name_dropping_kernel_over_a_nullable_key_is_collected`] -/// is the same fixtures over a scan whose keys can spell that pair. -#[test] -fn a_name_dropping_kernel_over_one_metric_keeps_its_partitions() { - let out = grid(0, 1000, 2); - let single = [Some(Arc::::from("some_metric"))]; - let kept = |name: &str, root: NativeNode| { - let tree = passed_over(root, &named_scan_tags(), &single); - assert_eq!(0, merges(&tree), "`{name}` cannot alias: {tree}"); - assert!( - !matches!(tree.root().expr, NativeExpr::Merge { .. }), - "`{name}` hands over one series per label set already: {tree}" - ); - }; - // A math map. - kept("abs", abs_map(selected(&out))); - // A histogram map. - kept("histogram_count", histogram_count(selected(&out))); - // A parameterized math kernel. - kept( - "clamp_min", - clamp_min(selected(&out), scalar(0.0, Arc::clone(&out))), - ); - // An arithmetic scalar broadcast, `x * 2`. - kept( - "x * 2", - binary_over( - selected(&out), - scalar(2.0, Arc::clone(&out)), - Arc::clone(&out), - ), - ); -} - -/// The same kernels over a binding that carries `__type__` or `__unit__` as a tag are collected at -/// the root. -/// -/// Those two go with the name when a kernel drops it, so two series that differ only there meet once -/// the drop removes them, whatever the scan's key columns read otherwise: this is the labels half -/// failing, not the key half. It is asked before the key columns are. -#[test] -fn a_name_dropping_kernel_over_a_metadata_tag_is_collected() { - let out = grid(0, 1000, 2); - let single = [Some(Arc::::from("some_metric"))]; - let collected = |name: &str, tags: &[Vec], root: NativeNode| { - let tree = passed_over(root, tags, &single); - assert_eq!(1, merges(&tree), "`{name}` drops the name: {tree}"); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "`{name}` hands out one series per label set: {tree}" - ); - }; - for metadata in ["__type__", "__unit__"] { - let scan = vec![tags(&["host", "__name__", metadata])]; - collected("abs", &scan, abs_map(selected(&out))); - collected("histogram_count", &scan, histogram_count(selected(&out))); - collected( - "clamp_min", - &scan, - clamp_min(selected(&out), scalar(0.0, Arc::clone(&out))), - ); - collected( - "x * 2", - &scan, - binary_over( - selected(&out), - scalar(2.0, Arc::clone(&out)), - Arc::clone(&out), - ), - ); - } -} - -/// A name-dropping kernel above another one asks for no collection of its own, and neither does a -/// broadcast above a window that drops the name: `rate(x[5m]) * 2` plans one window and one -/// broadcast, and `abs(abs(rate(x[5m])))` one window and two maps, with no merge in either. -/// -/// That is the answer over a scan whose key columns always decode a value. Over one that can hold -/// `NULL` in one series and `''` in the other, the drop the labels cannot prove is collected - but -/// only once, at the bottom of the chain: the merge answers the empty-value half of every drop above -/// it, because it hands over one series per label set under the comparison that treats an empty -/// value as no label. Each extra merge would buffer its whole input to collect a pair the drop above -/// it cannot fold. The labels half is a different matter and is still asked - see -/// [`a_merge_below_a_drop_answers_only_the_empty_values`]. -#[test] -fn a_drop_above_a_name_dropping_kernel_needs_no_collection() { - let out = grid(0, 1000, 2); - // What the planner states for a scan of one metric: its tag columns and the name the binding - // gives every series it hands over. - let scan = [tags(&["host", "zone", "__name__"])]; - let single = [Some(Arc::::from("some_metric"))]; - let rate = || window_over(raw_input(0), None, Arc::clone(&out)); - // `rate(x[5m]) * 2`: the broadcast drops the metadata labels the window dropped already. - let broadcast = binary_over(rate(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)); - // `abs(abs(rate(x[5m])))`: two math kernels above the window, neither of which can fold a pair - // the window left distinct. - let nested = abs_map(abs_map(rate())); - for (query, root) in [ - ("rate(x[5m]) * 2", broadcast), - ("abs(abs(rate(x[5m])))", nested), - ] { - let tree = passed_over(root, &scan, &single); - assert_eq!(0, merges(&tree), "`{query}` plans a merge: {tree}"); - } -} - -/// The same two queries over a scan whose keys can spell one series as `NULL` and another as `''`: -/// the drop is collected once, at the bottom of the chain, and the drops above it cost nothing. -/// -/// One merge is what a nullable key column costs a statement, whatever the length of the chain above -/// it. The merge sits above the window - the bottom-most drop - so every map and broadcast above it -/// is asked nothing: what they read is already one series per canonical label set. -#[test] -fn a_nullable_key_column_collects_a_chain_of_drops_once() { - let out = grid(0, 1000, 2); - // The same facts as the test above, with a key column that can hold `NULL` in one series and - // `''` in the other, which no labels can prove apart. - let scan = [tags(&["host", "zone", "__name__"])]; - let single = [Some(Arc::::from("some_metric"))]; - let rate = || window_over(raw_input(0), None, Arc::clone(&out)); - let broadcast = binary_over(rate(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)); - let nested = abs_map(abs_map(rate())); - for (query, root) in [ - ("rate(x[5m]) * 2", broadcast), - ("abs(abs(rate(x[5m])))", nested), - ] { - let tree = passed_over_nullable_keys(root, &scan, &single); - assert_eq!( - 1, - merges(&tree), - "`{query}` pays for the one merge its key columns need, not one per drop: {tree}" - ); - } -} - -/// A nullable key column collects the drop it is about, and that merge sits above the drop rather -/// than anywhere else in the chain: the window's answer is what the two stored series arrive at. -#[test] -fn a_nullable_key_column_collects_the_bottom_most_drop() { - let out = grid(0, 1000, 2); - let scan = [tags(&["host", "zone", "__name__"])]; - let single = [Some(Arc::::from("some_metric"))]; - // `abs(abs(rate(x[5m])))` over the nullable key: the merge is the window's consumer, and both - // maps above it read what it hands over. - let tree = passed_over_nullable_keys( - abs_map(abs_map(window_over(raw_input(0), None, Arc::clone(&out)))), - &scan, - &single, - ); - assert_eq!(1, merges(&tree), "{tree}"); - let NativeExpr::Map { input, .. } = &tree.root().expr else { - panic!("the root stays the outer map: {tree}"); - }; - let NativeExpr::Map { input, .. } = &input.expr else { - panic!("the two maps are not collected apart: {tree}"); - }; - let NativeExpr::Merge { input } = &input.expr else { - panic!("the merge sits above the window: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Window { .. }), - "the merge sits above the window, not below it: {tree}" - ); -} - -/// A merge below a drop answers the empty-value half of the proof only: a merge keeps two series -/// that differ in `__name__`, `__type__` or `__unit__`, and removing those is exactly what a -/// metadata drop does. Two metrics whose ordinary labels are equal are therefore still collected -/// above a merge, one merge per collision. -/// -/// A selector that discovered several metrics is that shape: each metric's window is collected for -/// the name it drops, and the map above the merge is collected again for the name the merge kept. -#[test] -fn a_merge_below_a_drop_answers_only_the_empty_values() { - let out = grid(0, 1000, 2); - let scan = [tags(&["host", "__name__"])]; - // The source may hand over several metrics, which is what leaves the labels half unproved above - // the window's merge. - let tree = passed_over( - abs_map(window_over(raw_input(0), None, Arc::clone(&out))), - &scan, - &[None], - ); - assert_eq!( - 2, - merges(&tree), - "the window's name drop and the map's are two collisions, and the merge between them \ - cannot fold the names they remove: {tree}" - ); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the map hands out: {tree}"); - }; - let NativeExpr::Map { input, .. } = &input.expr else { - panic!("the root stays the map: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Merge { .. }), - "the window's answer is collected below the map: {tree}" - ); -} - -/// A `Merge` hands over one series per canonical label set - the one that treats an empty label -/// value as no label - so no drop above it can fold two of its series by removing an empty value. -/// The kernels above a merge therefore need no collection of their own for the empty-value half, one -/// merge per collision rather than one per kernel that repeats the collapse; the labels half is a -/// different matter (`a_merge_below_a_drop_answers_only_the_empty_values`). -#[test] -fn a_drop_above_a_merge_needs_no_collection_of_its_own() { - let out = grid(0, 1000, 2); - let single = [Some(Arc::::from("some_metric"))]; - // `label_replace(x, "host", "$1", "host", "(.*)")`: every host is renamed to one value, so the - // rewrite folds the scan's series into one label set and the pass collects it there. - let aliasing = || rewrite(selected(&out), "host", "host"); - - // `label_replace(x, "host", "$1", "host", "(.*)") * 2`: the broadcast drops the metadata labels - // above a merge, which cannot fold two series the merge already told apart. - let tree = passed_over( - binary_over(aliasing(), scalar(2.0, Arc::clone(&out)), Arc::clone(&out)), - &named_scan_tags(), - &single, - ); - assert_eq!(1, merges(&tree), "`label_replace(...) * 2`: {tree}"); - let NativeExpr::Binary { left, .. } = &tree.root().expr else { - panic!("the root stays the broadcast: {tree}"); - }; - let NativeExpr::Merge { input } = &left.expr else { - panic!("the vector side is handed one series per label set: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Map { .. }), - "the merge sits above the rewrite, and no second one above the match: {tree}" - ); - - // `abs(abs(rate(label_replace(...)[5m])))`: the rewrite's collision is the one collection, and - // neither map above it asks for another - their own name drop is over series a merge holds. - let tree = passed_over( - abs_map(abs_map(window_over( - aliasing(), - Some(Arc::clone(&out)), - Arc::clone(&out), - ))), - &named_scan_tags(), - &single, - ); - assert_eq!( - 1, - merges(&tree), - "`abs(abs(rate(label_replace(...)[5m])))`: {tree}" - ); - let NativeExpr::Map { input, .. } = &tree.root().expr else { - panic!("the root stays the outer map: {tree}"); - }; - let NativeExpr::Map { input, .. } = &input.expr else { - panic!("the two maps are not collected apart: {tree}"); - }; - let NativeExpr::Window { input, .. } = &input.expr else { - panic!("the inner map reads the window: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Merge { .. }), - "the one merge sits above the rewrite: {tree}" - ); - // A second pass changes nothing: the merge answers one series per label set, and nothing above - // it collects again. - let twice = merge_label_rewrites(&tree, &named_scan_tags(), &single, true).unwrap(); - assert_eq!(1, merges(&twice), "{twice}"); - assert_eq!(tree.to_string(), twice.to_string()); -} - -/// A collection below a merge still happens: two series the tree folds into one label set there are -/// collected by the merge the pass puts above the rewrite, and the drop above them asks for nothing -/// more - the merge is what makes the two arrive as one. -#[test] -fn a_collision_below_a_merge_is_collected_there() { - let out = grid(0, 1000, 2); - let single = [Some(Arc::::from("some_metric"))]; - // `rate(label_replace(x, "host", "$1", "host", "(.*)")[5m])`: every host is renamed to one - // value, so the rewrite folds the scan's series into one label set - below the window, whose - // `rate` drops the metric name on top of that. - let tree = passed_over( - window_over( - rewrite(selected(&out), "host", "host"), - Some(Arc::clone(&out)), - Arc::clone(&out), - ), - &named_scan_tags(), - &single, - ); - assert_eq!( - 1, - merges(&tree), - "the rewrite's own collection is what collects the collision: {tree}" - ); - let NativeExpr::Window { input, .. } = &tree.root().expr else { - panic!("the root stays the window: {tree}"); - }; - let NativeExpr::Merge { input } = &input.expr else { - panic!("the window reads the collected rewrite: {tree}"); - }; - assert!( - matches!(input.expr, NativeExpr::Map { .. }), - "the merge sits above the rewrite, not above the window: {tree}" - ); - let twice = merge_label_rewrites(&tree, &named_scan_tags(), &single, true).unwrap(); - assert_eq!(1, merges(&twice), "{twice}"); - assert_eq!(tree.to_string(), twice.to_string()); -} - -/// A name write is collected even when a merge already holds the series below it: rewriting a name -/// can still put two metrics whose ordinary labels are equal under one identity, which is the rule -/// the kernels' name-drop proof states. -/// -/// `abs(label_replace(label_replace(x, "__name__", "$1", "host", "(.*)"), "host", "", "", ""))` -/// deletes the host the name was derived from, so the name is the only label still telling two -/// series apart. `single_metric_name` in `native::plan` is what refuses to carry a written name: it -/// would otherwise hand the scan's own name back, and a binding whose key columns always decode a -/// value would then make the drop look injective. -#[test] -fn a_written_name_is_not_the_name_the_scan_binds() { - let out = grid(0, 1000, 2); - // The scan reads one metric, so nothing but the written name is left to tell two of its series - // apart. - let single = [Some(Arc::::from("some_metric"))]; - let rewritten = map_ops( - map_ops( - selected(&out), - vec![MapOp::Label(Arc::new( - LabelRewrite::replace("__name__", "$1", "host", "(.*)").unwrap(), - ))], - ), - vec![MapOp::Label(Arc::new( - LabelRewrite::replace("host", "", "", "").unwrap(), - ))], - ); - let tree = passed_over(abs_map(rewritten), &named_scan_tags(), &single); - // The rewrite of the name and the rewrite of the `host` it was derived from are collected in - // their own right, and the root merge is the `abs` above them - the node whose answer the - // destination check decides. - assert_eq!(3, merges(&tree), "{tree}"); - let NativeExpr::Merge { input: abs_node } = &tree.root().expr else { - panic!("the root collects what `abs` hands out: {tree}"); - }; - let NativeExpr::Map { input: below, op } = &abs_node.expr else { - panic!("the collected node is the map above the rewrites: {tree}"); - }; - assert!( - op.ops.iter().any(|op| matches!(op, MapOp::Math(_))), - "the root merge sits above `abs`, not above the rewrites: {tree}" - ); - let NativeExpr::Merge { input: delete } = &below.expr else { - panic!("`abs` reads the collected rewrite: {tree}"); - }; - let NativeExpr::Map { - input: name_write, .. - } = &delete.expr - else { - panic!("the merge below `abs` sits above the delete: {tree}"); - }; - assert!( - matches!(name_write.expr, NativeExpr::Merge { .. }), - "the name write is collected below the delete: {tree}" - ); -} - -#[test] -fn a_timestamp_is_collected_by_its_consumer() { - let out = grid(0, 1000, 2); - // `timestamp` reports a sample's own time under the metric name dropped, so a caller that says - // nothing about the scan's metric collects its answer at the root as well. - let stamped = || NativeNode { - expr: NativeExpr::Timestamp { - input: Box::new(selected(&out)), - }, - output: evaluated(fields(1), Arc::clone(&out)), - }; - let tree = passed(stamped(), &named_scan_tags()); - assert_eq!(1, merges(&tree)); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "the root collects what the timestamp hands out: {tree}" - ); - - // The scan reads one metric, so the name it drops told none of its series apart: the timestamp - // keeps every partition. - let tree = passed_over( - stamped(), - &named_scan_tags(), - &[Some(Arc::from("some_metric"))], - ); - assert_eq!(0, merges(&tree), "{tree}"); - assert!( - matches!(tree.root().expr, NativeExpr::Timestamp { .. }), - "the root stays the timestamp: {tree}" - ); -} - -/// The three nodes that drop the metadata labels on their own are judged by the labels: one metric -/// name over every series, and no metadata tag among them. -/// -/// A name-dropping window, `timestamp(x)` and a `timestamp` over an evaluated input all drop the -/// metadata labels. A source bound to one metric names every series it hands over, so that name told -/// none of them apart and the drop is proved; a source that may hand over several is collected, and -/// so is one that carries `__type__` or `__unit__` among its tags. A window that keeps the name -/// drops nothing and hands its input's identity on. -#[test] -fn a_name_dropping_node_is_proved_by_the_labels_alone() { - let out = grid(0, 1000, 2); - let named = named_scan_tags(); - let single = [Some(Arc::::from("some_metric"))]; - let collected = - |name: &str, tags: &[Vec], metrics: &[Option>], root: NativeNode| { - let tree = passed_over(root, tags, metrics); - assert_eq!( - 1, - merges(&tree), - "`{name}` drops the metadata labels: {tree}" - ); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "the root collects `{name}`: {tree}" - ); - }; - let kept = |name: &str, root: NativeNode| { - let tree = passed_over(root, &named, &single); - assert_eq!(0, merges(&tree), "`{name}` cannot alias: {tree}"); - assert!( - !matches!(tree.root().expr, NativeExpr::Merge { .. }), - "the root stays `{name}`: {tree}" - ); - }; - // `timestamp(x)` over the scan, `rate(x[5m])`, and `timestamp(rate(x[5m]))` over an evaluated - // input: the three nodes that drop the metadata labels themselves. - let stamps = || selection(raw_input(0), Arc::clone(&out), InstantValue::Timestamp); - let drops = || window_over(raw_input(0), None, Arc::clone(&out)); - let evaluated_stamps = || NativeNode { - expr: NativeExpr::Timestamp { - input: Box::new(drops()), - }, - output: evaluated(fields(1), Arc::clone(&out)), - }; - - // The labels hold: one metric names every series, and no metadata tag is among the tags. - kept("timestamp", stamps()); - kept("rate", drops()); - kept("timestamp(rate)", evaluated_stamps()); - // A source that may hand over several metrics is not proved: the name it drops is what tells - // two of them apart. - collected("timestamp", &named, &[None], stamps()); - collected("rate", &named, &[None], drops()); - collected("timestamp(rate)", &named, &[None], evaluated_stamps()); - // A binding that keys a series by a metadata label is collected: that label goes with the name, - // so two series that differ only there meet. - for metadata in ["__type__", "__unit__"] { - let mut typed = named.clone(); - typed[0].push(metadata.to_string()); - collected("timestamp", &typed, &single, stamps()); - collected("rate", &typed, &single, drops()); - collected("timestamp(rate)", &typed, &single, evaluated_stamps()); - } - // A window that keeps the name drops nothing, so nothing is asked of it. - let tree = passed_over( - keeping_window_over(raw_input(0), None, Arc::clone(&out)), - &named, - &single, - ); - assert_eq!(0, merges(&tree), "`last_over_time` keeps the name: {tree}"); - assert!( - matches!(tree.root().expr, NativeExpr::Window { .. }), - "the root stays the name-keeping window: {tree}" - ); -} - #[test] fn info_collects_the_side_that_aliases() { let out = grid(0, 1000, 2); @@ -1494,7 +780,7 @@ fn info_collects_the_side_that_aliases() { } #[test] -fn a_scalar_broadcast_collects_its_own_name_drop() { +fn a_scalar_side_of_a_broadcast_is_not_collected() { let out = grid(0, 1000, 2); let tree = passed( binary_over( @@ -1504,23 +790,7 @@ fn a_scalar_broadcast_collects_its_own_name_drop() { ), &scan_tags(), ); - // `x + 2` drops the metadata labels, so what it hands out is collected - above the match, not - // beside it - and neither operand is collected on its own. - assert_eq!(1, merges(&tree), "{tree}"); - let NativeExpr::Merge { input } = &tree.root().expr else { - panic!("the root collects what the broadcast hands out: {tree}"); - }; - let NativeExpr::Binary { left, right, .. } = &input.expr else { - panic!("the root stays the broadcast"); - }; - assert!( - matches!(left.expr, NativeExpr::Scalar(_)), - "a constant holds no label set" - ); - assert!( - !matches!(right.expr, NativeExpr::Merge { .. }), - "the vector side hands its series over as it reads them" - ); + assert_eq!(0, merges(&tree), "a constant holds no label set"); } #[test] @@ -1543,7 +813,7 @@ fn aggregate_state_is_not_collected() { let tree = passed(state, &scan_tags()); assert_eq!(0, merges(&tree)); assert!(matches!(tree.root().output, NativeType::AggregateState(_))); - let twice = merge_label_rewrites(&tree, &scan_tags(), &[], true).unwrap(); + let twice = merge_label_rewrites(&tree, &scan_tags()).unwrap(); assert_eq!(tree.to_string(), twice.to_string()); } @@ -1567,8 +837,7 @@ fn the_pass_is_idempotent() { ), &scan_tags(), ); - let twice = - merge_label_rewrites(&once, &scan_tags(), &[], true).expect("a collected tree is legal"); + let twice = merge_label_rewrites(&once, &scan_tags()).expect("a collected tree is legal"); assert_eq!(1, merges(&twice)); assert_eq!(once.to_string(), twice.to_string()); } diff --git a/src/promql/src/native/execute.rs b/src/promql/src/native/execute.rs index 8421e6f4052..55c81d5c664 100644 --- a/src/promql/src/native/execute.rs +++ b/src/promql/src/native/execute.rs @@ -18,7 +18,8 @@ //! tree builds independent executions. A caller registers the tree's per-node metric handles into //! its own metrics set once, before the execution starts. -use std::collections::{HashSet, VecDeque}; +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use common_query::promql_annotations::PromqlAnnotationCollector; @@ -37,8 +38,8 @@ use datatypes::series::{SeriesLabels, SeriesOrdering}; use crate::aggregate; use crate::grid_series::{ - GridSample, GridSeriesBuilder, fields_bytes, merge_same_labelset, owned, series_bytes, - table_bytes, + GridSample, GridSeriesBuilder, LabelSet, fields_bytes, label_set, merge_same_labelset, owned, + series_bytes, table_bytes, }; use crate::histogram_eval::float_histogram::FloatHistogram; use crate::histogram_eval::{ @@ -63,6 +64,9 @@ use crate::native::plan::{ BoundInstant, BoundScalar, BoundSetOp, BoundVectorOp, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, ParameterizedOp, unifies_its_own_contributions, }; +#[cfg(test)] +pub(crate) use crate::native::runs::absorb_rebuilds; +use crate::native::runs::{Run, absorb}; use crate::native::state::{StateChunk, state_bytes}; use crate::range_eval::RangeEvaluator; use crate::select::{self, SortOrder}; @@ -536,15 +540,6 @@ impl MergeInput { pub(super) struct NodeBuffer { metrics: Arc, reservation: MemoryReservation, - /// What this node keeps for the grid timestamps it materialized, folded into every - /// re-estimate of its own figure; see [`NodeBuffer::reserve_grid`]. - /// - /// `None` means this node has not reserved a grid yet and `Some(bytes)` that it has: what the - /// record says is that the reservation was made, which is not the same statement as the figure - /// it ended at. A grid always has at least one point today -- `GridSpec` refuses a length of - /// zero -- so the two happen to coincide, but a reservation of no bytes would still be a - /// reservation made, and reading the figure would make it one made per call. - grid: Option, } impl NodeBuffer { @@ -561,46 +556,9 @@ impl NodeBuffer { Self { metrics, reservation: consumer.register(pool), - grid: None, } } - /// Reserves the timestamps of `grid` before this node reads them, once for the execution. - /// - /// An [`EvalGrid`] allocates its timestamps on first access and keeps them for as long as the - /// plan holds the grid, which outlives the chunk a node read them for. The node that triggers - /// that allocation therefore reserves the array and keeps the reservation: giving it back with - /// the workspace would leave the execution holding bytes no reservation covers, which is what a - /// grid of a million deferred points -- the reason the array is deferred at all -- would go - /// unnoticed through. Only the node that materializes it reserves it: a grid that is already - /// materialized, whether the plan built it eagerly or a node below or beside this one needed it - /// first, was charged to whoever materialized it, so this node grows by nothing. - /// - /// The array is reserved once per node, not once per call: a node is handed its input one chunk - /// at a time and reaches the grid on every chunk it is handed, and a collecting window reaches - /// it again when it flushes what it collected -- which is the call that reads the timestamps, - /// because until then nothing on this node looks at them. Reserving again on each of those - /// calls charges the pool one array per chunk for the one array the execution holds, so a series - /// arriving in `n` pieces would need `n + 1` arrays' worth of budget and exhaust a pool sized - /// for the query over a chunk count that says nothing about what the node keeps. What short - /// circuits the later calls is therefore the reservation this node made (`grid`), not the array - /// having been materialized: it cannot stand in for it, being false exactly while a collecting - /// node is charging for it. - pub(super) fn reserve_grid(&mut self, grid: &EvalGrid) -> Result<()> { - if self.grid.is_some() || grid.is_materialized() { - return Ok(()); - } - let len = usize::try_from(grid.spec().len).map_err(|_| { - DataFusionError::Execution("grid length does not fit memory size".to_string()) - })?; - let bytes = len.checked_mul(std::mem::size_of::()).ok_or_else(|| { - DataFusionError::Execution("grid timestamp size overflow".to_string()) - })?; - self.grow(bytes)?; - self.grid = Some(bytes); - Ok(()) - } - /// Reserves `bytes` beside what this node already holds. pub(super) fn grow(&mut self, bytes: usize) -> Result<()> { self.reservation.try_grow(bytes)?; @@ -609,12 +567,9 @@ impl NodeBuffer { } /// Reserves exactly `bytes`, which is what this node holds from now on: a node that consumes its - /// input chunk by chunk is set from the state that is left, not grown per chunk. What a grid's - /// array takes is not part of that figure: the node keeps it for as long as it lives, so it - /// stays reserved on top of what the caller measured. + /// input chunk by chunk is set from the state that is left, not grown per chunk. pub(super) fn resize(&mut self, bytes: usize) -> Result<()> { - self.reservation - .try_resize(bytes.saturating_add(self.grid.unwrap_or(0)))?; + self.reservation.try_resize(bytes)?; self.record(); Ok(()) } @@ -657,7 +612,51 @@ impl Drop for NodeBuffer { } } -/// The bytes a metric name holds: its name and each tag's name and value, one row per label. +/// The positions one node has handed out, per output label set and field. +/// +/// A node that rewrites labels -- a `Map` dropping the metric name, a `Window` dropping the name and the +/// empty-valued labels -- can turn two input series into one label set, and the node above would read the +/// two chunks as one series. Recording which positions of each field were already handed out lets a second +/// series onto a set be refused before it is evaluated. +/// +/// Occupancy is per field and per position, and a chunk holding no value occupies nothing. A set is never +/// retired: `is_last` closes a source series, not a label set. The table is one node's own. +#[derive(Default)] +struct Emitted { + sets: HashMap, + /// How many run vectors, runs and label payload bytes every entry holds, folded in as the entries + /// change rather than by walking them, so [`Emitted::bytes`] is read in constant time. + counts: Counts, +} + +/// What the entries of an [`Emitted`] table hold, as running totals. +#[derive(Default)] +struct Counts { + /// The run vectors, one per field of one entry. + vectors: usize, + /// The occupied runs in every vector. + runs: usize, + /// The payload of every label set: the bytes of its names and values with one row per label. + payload: usize, +} + +/// What one node has handed out under one output label set. +#[derive(Default)] +struct History { + /// The occupied runs of each field, in position order and disjoint. + fields: Vec>, +} + +/// The bytes a label set holds: each label's name and value with the pair that refers to them. +fn label_bytes(labels: &LabelSet) -> usize { + labels + .iter() + .map(|(name, value)| std::mem::size_of::<(Arc, Arc)>() + name.len() + value.len()) + .sum() +} + +/// The bytes a metric name holds: its name and each tag's name and value, one row per label, which is +/// the convention [`label_bytes`] keeps. fn metric_bytes(metric: &SeriesLabels) -> usize { metric.name().map_or(0, |name| name.len()) + std::mem::size_of_val(metric.tags()) @@ -668,6 +667,137 @@ fn metric_bytes(metric: &SeriesLabels) -> usize { .sum::() } +impl Emitted { + /// Records what `chunk` occupies, rejecting it against what this node already handed out. + /// + /// Two values of one field at one position is the only thing refused: a second series that fills + /// only the positions the first one left empty merges into it. + fn occupy(&mut self, chunk: &EvaluatedChunk) -> Result<()> { + let labels = label_set(&chunk.series.metric); + let occupied: Vec> = chunk + .series + .fields + .iter() + .enumerate() + .map(|(index, field)| occupied_runs(index, field, &chunk.series.time)) + .collect::>()?; + let empty = occupied.iter().all(Vec::is_empty); + if let Some(history) = self.sets.get(&labels) { + for (index, runs) in occupied.iter().enumerate() { + let stored = history.fields.get(index).map_or(&[][..], Vec::as_slice); + for run in runs { + // The stored runs are sorted and disjoint, so only the one holding `run.start` + // or the one just before it can share a position. + let at = stored.partition_point(|stored| stored.end <= run.start); + let previous = stored.get(at).filter(|stored| stored.overlaps(*run)); + if let Some(previous) = previous { + return Err(occupied_position(&labels, index, run, previous)); + } + } + } + } + if empty { + // A chunk holding no value records nothing and closes nothing: a series that answered + // empty leaves every earlier series on this label set exactly where it was. + return Ok(()); + } + let fields = chunk.series.fields.len(); + let history = match self.sets.entry(labels) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + self.counts.payload += label_bytes(entry.key()); + entry.insert(History::default()) + } + }; + if history.fields.len() < fields { + self.counts.vectors += fields - history.fields.len(); + history.fields.resize_with(fields, Vec::new); + } + for (index, runs) in occupied.into_iter().enumerate() { + let stored = &mut history.fields[index]; + let before = stored.len(); + absorb(stored, &runs); + self.counts.runs = self.counts.runs - before + stored.len(); + } + Ok(()) + } + + /// An estimate of the bytes this table holds, which the node reserves for. + /// + /// Read from running totals, so it is constant time; the map's buckets and a vector's slack are + /// left out. + fn bytes(&self) -> usize { + let slots = + self.sets.len() * (std::mem::size_of::() + std::mem::size_of::()); + slots + + self.counts.vectors * std::mem::size_of::>() + + self.counts.runs * std::mem::size_of::() + + self.counts.payload + } +} + +/// The runs of positions holding a value in field `index`, as the axis `time` places its rows. +/// +/// A missing sample holds no value and a hole between rows is not a position, so a run ends where a hole +/// begins. Every field must hold one sample per row the axis places. +fn occupied_runs(index: usize, field: &Samples, time: &TimeAxis) -> Result> { + let mut runs: Vec = Vec::new(); + let mut open: Option = None; + let mut previous: Option = None; + let mut cursor = field.cursor(); + for (position, row) in time.positions() { + // The axis names the absolute position of every row, so the hole between two rows is + // visible as a jump in `position` while no row of this chunk sits in it. + if let Some(previous) = previous + && position != previous + 1 + && let Some(start) = open.take() + { + runs.push(Run { + start, + end: previous + 1, + }); + } + let sample = cursor + .at(row) + .ok_or_else(|| misaligned_field(index, time, field))?; + let present = !matches!(sample, SampleRef::Missing); + match (present, open) { + (true, None) => open = Some(position), + (false, Some(start)) => { + runs.push(Run { + start, + end: position, + }); + open = None; + } + _ => {} + } + previous = Some(position); + } + if let Some(start) = open { + // A run still open at the last row ends just past it, not where the field's row count says: + // the rows need not be adjacent to the positions above them. + runs.push(Run { + start, + end: previous.map_or(start, |last| last + 1), + }); + } + Ok(runs) +} + +/// The rejection of a series whose axis and fields disagree on how many rows there are. +/// +/// [`Series`] keeps its fields public, so a struct-literal chunk can carry fewer samples than the +/// axis places rows. +fn misaligned_field(index: usize, time: &TimeAxis, field: &Samples) -> DataFusionError { + DataFusionError::Execution(format!( + "a series was handed to the node whose axis and fields are misaligned: the axis places {} \ + rows, but field {index} holds {} samples, so the axis names a row the field does not have", + time.len(), + field.len(), + )) +} + /// What a map node applies. struct MapRuntime { ops: Arc<[MapOp]>, @@ -2064,11 +2194,9 @@ impl BroadcastRuntime { /// Whether the operation rewrites the label set of what it hands out. /// - /// Arithmetic and a `bool` comparison drop the metric name, so two input series can become one - /// label set under this node; a filtering comparison keeps the label set it read. The answer is - /// read off [`Self::op`], so an operation and its label policy cannot disagree. Collection is not - /// this node's work: a collision the rewritten set creates is what a canonical - /// [`NativeExpr::Merge`] above it refuses. + /// Arithmetic and a `bool` comparison drop the metric name, so two input series can become one label + /// set under this node; a filtering comparison keeps the label set it read. The answer is read off + /// [`Self::op`], so an operation and its label policy cannot disagree. fn drops_labels(&self) -> bool { match self.op { BroadcastOp::Arithmetic(_) | BroadcastOp::Compare(Comparison::Bool(_)) => true, @@ -2666,10 +2794,11 @@ enum RuntimeOp { /// State shared by the stateful operators. /// -/// The queue must precede the buffer so it drops before its reservation. +/// The queue and emitted history must precede the buffer so they drop before its reservation. struct NodeState { queued: VecDeque<(NativeChunk, usize)>, queued_tables: usize, + emitted: Option, buffer: NodeBuffer, } @@ -2763,6 +2892,7 @@ impl NodeState { Self { queued: VecDeque::new(), queued_tables: 0, + emitted: None, buffer: NodeBuffer::new(name, metrics, pool), } } @@ -2789,7 +2919,6 @@ impl RuntimeNode { binary_before: &mut usize, location: String, duplicate_evidence: Option>, - sources: &SourceFacts, ) -> Result { stacker::maybe_grow(NATIVE_STACK_RED_ZONE, NATIVE_STACK_GROWTH, || { Self::build_within_stack( @@ -2799,7 +2928,6 @@ impl RuntimeNode { binary_before, location, duplicate_evidence, - sources, ) }) } @@ -2811,7 +2939,6 @@ impl RuntimeNode { binary_before: &mut usize, location: String, duplicate_evidence: Option>, - sources: &SourceFacts, ) -> Result { let metrics = Arc::new(NodeMetrics::default()); let op = match &node.expr { @@ -2858,7 +2985,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; RuntimeOp::ScalarOf { input: Box::new(child), @@ -2905,7 +3031,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; RuntimeOp::VectorOf { input: Box::new(child), @@ -2927,7 +3052,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let input_schema = input.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution( @@ -2938,11 +3062,15 @@ impl RuntimeNode { input: Box::new(child), instant: InstantRuntime::try_new(&input_schema, op)?, state: { - NodeState::new( + let mut state = NodeState::new( format!("native.instant at {location}"), Arc::clone(&metrics), pool, - ) + ); + if op.value == InstantValue::Timestamp { + state.emitted = Some(Emitted::default()); + } + state }, } } @@ -2954,7 +3082,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let schema = node.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("merge does not produce samples".to_string()) @@ -2978,23 +3105,18 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let input_schema = input.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("window consumes non-sample input".to_string()) })?; - // The child reports fragments only when it cannot prove the labels it drops tell its - // series apart, which is the same question the canonical pass asked before it - // decided this window needs no merge above it. - let unification = - unifies_its_own_contributions(input, &sources.tags, &sources.metrics); - let merge = matches!(unification, Some(false)) + let merge = matches!(unifies_its_own_contributions(input), Some(false)) .then(|| MergeInput::new(Arc::clone(&input_schema))); - let state = NodeState::new( + let mut state = NodeState::new( format!("native.window at {location}"), Arc::clone(&metrics), pool, ); + state.emitted = Some(Emitted::default()); RuntimeOp::Window { input: Box::new(child), // The collector arrives through `set_annotations` before the first pull; @@ -3017,16 +3139,16 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let schema = node.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("map does not produce samples".to_string()) })?; - let state = NodeState::new( + let mut state = NodeState::new( format!("native.map at {location}"), Arc::clone(&metrics), pool, ); + state.emitted = Some(Emitted::default()); RuntimeOp::Map { input: Box::new(child), map: MapRuntime { @@ -3048,7 +3170,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; // A function of two scalars answers a scalar, which has one unlabelled field on // the grid it is evaluated on; every other function here answers its input's own @@ -3087,15 +3208,15 @@ impl RuntimeNode { binary_before, format!("{location}.parameter{index}"), duplicate_evidence.clone(), - sources, ) }) .collect::>>()?; - let state = NodeState::new( + let mut state = NodeState::new( format!("native.map at {location}"), Arc::clone(&metrics), pool, ); + state.emitted = Some(Emitted::default()); RuntimeOp::ParameterizedMap { input: Box::new(child), parameters: nodes, @@ -3119,16 +3240,16 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let schema = node.output.samples().cloned().ok_or_else(|| { DataFusionError::Execution("timestamp does not produce samples".to_string()) })?; - let state = NodeState::new( + let mut state = NodeState::new( format!("native.timestamp at {location}"), Arc::clone(&metrics), pool, ); + state.emitted = Some(Emitted::default()); RuntimeOp::Timestamp { input: Box::new(child), schema, @@ -3167,7 +3288,6 @@ impl RuntimeNode { binary_before, format!("{location}.left"), duplicate_evidence.clone(), - sources, )?; let right_node = Self::build( right, @@ -3176,7 +3296,6 @@ impl RuntimeNode { binary_before, format!("{location}.right"), duplicate_evidence.clone(), - sources, )?; match (binary, broadcast) { (Some(binary), None) => RuntimeOp::Binary { @@ -3190,11 +3309,18 @@ impl RuntimeNode { ), }, (None, Some(broadcast)) => { - let state = NodeState::new( + let vector = if broadcast.vector_left { left } else { right }; + let mut state = NodeState::new( format!("native.binary at {location}"), Arc::clone(&metrics), pool, ); + // Only an operation that rewrites the label set is where two input series + // can become one, and only one that makes an identity its vector side has + // not already checked needs a table of its own. + if broadcast.drops_labels() && !elides_emitted_table(&broadcast, vector) { + state.emitted = Some(Emitted::default()); + } RuntimeOp::Broadcast { left: Box::new(left_node), right: Box::new(right_node), @@ -3232,7 +3358,6 @@ impl RuntimeNode { binary_before, format!("{location}.left"), duplicate_evidence.clone(), - sources, )?; let right_node = Self::build( right, @@ -3241,7 +3366,6 @@ impl RuntimeNode { binary_before, format!("{location}.right"), duplicate_evidence.clone(), - sources, )?; RuntimeOp::Binary { left: Box::new(left_node), @@ -3281,7 +3405,6 @@ impl RuntimeNode { binary_before, format!("{location}.base"), duplicate_evidence.clone(), - sources, )?; let info_node = Self::build( info, @@ -3290,7 +3413,6 @@ impl RuntimeNode { binary_before, format!("{location}.info"), duplicate_evidence.clone(), - sources, )?; RuntimeOp::Binary { left: Box::new(base_node), @@ -3325,7 +3447,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; RuntimeOp::Aggregate { input: Box::new(child), @@ -3355,7 +3476,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let parameter = Self::build( parameter, @@ -3364,7 +3484,6 @@ impl RuntimeNode { binary_before, format!("{location}.parameter"), duplicate_evidence.clone(), - sources, )?; let state = NodeState::new( format!("native.vector at {location}"), @@ -3419,7 +3538,6 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let mut built = Vec::with_capacity(parameters.len()); for (index, parameter) in parameters.iter().enumerate() { @@ -3430,7 +3548,6 @@ impl RuntimeNode { binary_before, format!("{location}.parameter{index}"), duplicate_evidence.clone(), - sources, )?; built.push(Some(runtime)); } @@ -3474,15 +3591,9 @@ impl RuntimeNode { binary_before, format!("{location}.input"), duplicate_evidence.clone(), - sources, )?; let limitk = matches!(op, BoundVectorOp::LimitK { .. }); - if limitk - && matches!( - unifies_its_own_contributions(input, &sources.tags, &sources.metrics), - Some(false) - ) - { + if limitk && matches!(unifies_its_own_contributions(input), Some(false)) { vector.merge_input = true; } RuntimeOp::Vector { @@ -3839,12 +3950,6 @@ impl RuntimeNode { state, } => { if !std::mem::replace(produced, true) { - // `time()` and the date functions read every point of the grid, so this node is - // what materializes a deferred grid's timestamps. - state - .buffer - .reserve_grid(grid) - .map_err(|error| state.context(error))?; let schema = scalar_schema(Arc::clone(grid)); state .produce(grid, || match date { @@ -4268,7 +4373,7 @@ impl RuntimeNode { .settle_vector(operation) .map_err(|error| state.context(error))?; // Finish windows before dropping the consumed subtree; its children may still - // hold charged buffers or scalar series even after the scalar was read. + // hold charged emitted history or scalar series even after the scalar was read. parameter_node.finish_windows()?; *parameter = None; } @@ -4801,13 +4906,9 @@ impl RuntimeNode { .. } => { let child = input.finish_windows(); - // Finishing evaluates the points still owed, which reads the grid's timestamps. let own = window .take() - .map_or(Ok(()), |evaluator| { - state.buffer.reserve_grid(evaluator.grid())?; - RangeEvaluator::finish(evaluator) - }) + .map_or(Ok(()), RangeEvaluator::finish) .map_err(|error| state.context(error)); child.and(own) } @@ -4815,59 +4916,11 @@ impl RuntimeNode { } } -/// What a fused execution knows about the labels its inputs hand over. -/// -/// The planner binds the tree's inputs and knows two things about each: the complete label set it -/// reports, and - when the statement reads one table - the one metric every series it hands over is -/// named. [`unifies_its_own_contributions`] reads exactly those facts, so the collection a consumer -/// performs at run time is decided by the same proof the canonical pass placed its merges with. A -/// caller that knows nothing sets no facts, and every consumer then collects conservatively. -#[derive(Clone, Debug, Default)] -pub struct SourceFacts { - /// The complete runtime label set of each input, indexed by the `source_index` of its leaf. - tags: Vec>, - /// The one metric each input is bound to, indexed the same way; `None` when the input may hand - /// over several, which is what a statement reading several tables binds. - metrics: Vec>>, -} - -impl SourceFacts { - /// Says nothing about any input, so every decision stays conservative. - pub fn none() -> Self { - Self::default() - } - - /// One statement-level fact, repeated for each of `sources` inputs. - /// - /// A binding names one metric and one tag set for the node as a whole, and every scan below it - /// reads that one table, so the fact holds for each of them. A binding that names no metric - - /// the node reads one raw source per input - proves nothing, and the proof fails with it. - pub fn uniform(tags: Vec, metric: Option>, sources: usize) -> Self { - Self { - tags: vec![tags; sources], - metrics: vec![metric; sources], - } - } -} - impl NativeExecutionState { pub fn try_new( tree: &NativeTree, pool: Arc, duplicate_evidence: Option>, - ) -> Result { - Self::try_new_with_sources(tree, pool, duplicate_evidence, &SourceFacts::none()) - } - - /// Builds the execution over `tree`, judging a dropped label set by `sources`. - /// - /// The canonical pass placed its merges with the same facts, so a consumer that collects here - /// collects exactly where the pass decided a consumer has to - and nowhere else. - pub fn try_new_with_sources( - tree: &NativeTree, - pool: Arc, - duplicate_evidence: Option>, - sources: &SourceFacts, ) -> Result { let mut inputs = Vec::new(); let mut binary_before = 0usize; @@ -4878,7 +4931,6 @@ impl NativeExecutionState { &mut binary_before, "root".to_string(), duplicate_evidence.clone(), - sources, )?; // A constant root has no node that produces its value, so the state produces it. let scalar_root = match (&tree.root().expr, &tree.root().output) { @@ -5092,18 +5144,21 @@ impl NodeState { let output = match outcome { Ok(output) => output, Err(error) => { - self.buffer.resize(0)?; + self.buffer.resize(self.emitted_bytes())?; return Err(error); } }; if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); + // What it handed out is what the label sets of this node now hold: `map_chunk` has + // already rewritten the output's label set, so the label set of the output is the one + // two input series can share. + self.track(output)?; } - // The chunk is gone and this node holds none of it: a rewritten label set is not this - // node's to remember, and what two series sharing one would mean is a canonical merge's - // question. - self.buffer.resize(0)?; + // The chunk is gone and the label sets this node has handed out are what is left. + let emitted = self.emitted_bytes(); + self.buffer.resize(emitted)?; Ok(output) } @@ -5133,7 +5188,8 @@ impl NodeState { let outcome = match outcome { Ok(outcome) => outcome, Err(error) => { - self.buffer.resize(map.parameter_bytes)?; + self.buffer + .resize(self.emitted_bytes().saturating_add(map.parameter_bytes))?; return Err(error); } }; @@ -5151,9 +5207,11 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); + self.track(output)?; } - // What this node still holds is what it read for its parameters and nothing else. - self.buffer.resize(map.parameter_bytes)?; + let emitted = self.emitted_bytes(); + self.buffer + .resize(emitted.saturating_add(map.parameter_bytes))?; Ok(output) } @@ -5165,16 +5223,21 @@ impl NodeState { ) -> Result> { let positions = chunk.series.time.len(); let closing = chunk.is_last; - // The kernel reads the grid's timestamps, which materializes them when the grid is - // deferred. They stay materialized for the rest of the execution, so this node reserves - // them beside what it holds and keeps them there rather than treating them as a workspace - // it gives back. - if let Some(grid) = schema.eval_grid.as_deref() { - self.buffer.reserve_grid(grid)?; - } + // The kernel reads the grid's timestamps, which it materializes when they are deferred. + let grid_points = schema.eval_grid.as_ref().map_or(0, |grid| { + usize::try_from(grid.spec().len).unwrap_or(usize::MAX) + }); let workspace = built_series_workspace_bytes(&chunk.series)?; + let scratch = grid_points + .checked_mul(std::mem::size_of::()) + .and_then(|timestamps| timestamps.checked_add(workspace)) + .ok_or_else(|| { + DataFusionError::Execution( + "the workspace of timestamp does not fit one address space".to_string(), + ) + })?; self.buffer.grow(evaluated_bytes(&chunk))?; - self.buffer.grow(workspace)?; + self.buffer.grow(scratch)?; let labels = drop_metric_name(&chunk.series.metric)?; let metrics = Arc::clone(&self.buffer.metrics); let result = { @@ -5189,7 +5252,7 @@ impl NodeState { let result = match result { Ok(result) => result, Err(error) => { - self.buffer.resize(0)?; + self.buffer.resize(self.emitted_bytes())?; return Err(error); } }; @@ -5201,8 +5264,9 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); + self.track(output)?; } - self.buffer.resize(0)?; + self.buffer.resize(self.emitted_bytes())?; Ok(output) } @@ -5249,7 +5313,8 @@ impl NodeState { let output = match outcome { Ok(output) => output, Err(error) => { - self.buffer.resize(broadcast.scalar_bytes)?; + self.buffer + .resize(self.emitted_bytes().saturating_add(broadcast.scalar_bytes))?; return Err(error); } }; @@ -5260,9 +5325,17 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); + // Only an operation that rewrites the label set is where two input series can become one, and + // only that one keeps the table `track` records into. + if broadcast.drops_labels() { + self.track(output)?; + } } - // The chunk is gone and the scalar's value is what is left. - self.buffer.resize(broadcast.scalar_bytes)?; + // The chunk is gone: the label sets this node has handed out and the scalar's value are + // what is left. + let emitted = self.emitted_bytes(); + self.buffer + .resize(emitted.saturating_add(broadcast.scalar_bytes))?; Ok(output) } @@ -5328,9 +5401,6 @@ impl NodeState { merge: &mut Option, chunk: NativeChunk, ) -> Result> { - // A window reads the grid's timestamps to place every point it decides, which materializes - // a deferred grid; the array stays materialized, so the reservation for it is kept. - self.buffer.reserve_grid(window.grid())?; if let Some(merge) = merge.as_mut() { let NativeChunk::Evaluated(chunk) = chunk else { return Err(DataFusionError::Execution( @@ -5391,19 +5461,22 @@ impl NodeState { let output = match outcome { Ok(output) => output, Err(error) => { - self.buffer.resize(window.retained_bytes())?; + self.buffer + .resize(window.retained_bytes().saturating_add(self.emitted_bytes()))?; return Err(error); } }; if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); + self.track(output)?; } - // The chunk has been consumed: what this node still holds is what the evaluator kept of it, - // charged whether or not this chunk produced an answer: a chunk that yields nothing leaves - // the evaluator's samples where they were, and the figure is not charged per chunk consumed, - // which would make a fixed window over a long series read as an allocation failure. - self.buffer.resize(window.retained_bytes())?; + // The chunk has been consumed: what this node still holds is what the evaluator kept of it and the + // label sets it has already handed out. Neither is charged per chunk consumed, which would make a + // fixed window over a long series read as an allocation failure. + let retained = window.retained_bytes(); + let emitted = self.emitted_bytes(); + self.buffer.resize(retained.saturating_add(emitted))?; Ok(output) } @@ -5693,11 +5766,6 @@ impl NodeState { )); }; let (positions, closing) = (chunk.time.len(), chunk.is_last); - // A selection that answers with timestamps reads every point of the grid, which materializes - // a deferred grid's array; the reservation is kept because the array is. - if instant.selection.value != InstantValue::Sample { - self.buffer.reserve_grid(&instant.selection.output_grid)?; - } // The chunk is held from here on: a selection keeps every chunk of a series until the one // that closes it, so what this node reserves for is the series it has buffered. self.buffer.grow(raw_bytes(&chunk))?; @@ -5736,6 +5804,9 @@ impl NodeState { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); + if instant.selection.value == InstantValue::Timestamp { + self.track(output)?; + } } // What this node still holds is the chunks of the series it is selecting: a whole series is held // until it ends. The chunk just consumed and the gather's workspace are both gone, so this gives @@ -5746,7 +5817,11 @@ impl NodeState { /// Sets this node's reservation to the chunks of the series it is still gathering. fn settle_buffered(&mut self, instant: &InstantRuntime) -> Result<()> { - self.buffer.resize(instant.buffered_bytes()) + self.buffer.resize( + instant + .buffered_bytes() + .saturating_add(self.emitted_bytes()), + ) } /// Retains and validates a fragment for canonicalization. @@ -5803,9 +5878,12 @@ impl NodeState { chunk: EvaluatedChunk, ) -> Result> { self.collect_merge(merge, chunk)?; - // What this node holds is the fragments; its evaluator has read none of them yet. + // What this node holds is the fragments and the label sets it has already handed out; its + // evaluator has read none of them yet. let retained = window.retained_bytes(); - self.buffer.resize(retained.saturating_add(merge.bytes))?; + let emitted = self.emitted_bytes(); + self.buffer + .resize(retained.saturating_add(emitted).saturating_add(merge.bytes))?; Ok(None) } @@ -5827,8 +5905,6 @@ impl NodeState { // evaluator never read a series and the charge goes back. return self.buffer.shrink(merge.bytes); } - // The merged series is evaluated by the window, which reads the grid's timestamps. - self.buffer.reserve_grid(window.grid())?; // The workspace the merge allocates is charged before it runs, from the rows, segments and // fields of the fragments as they were collected. let workspace = merge.workspace_bytes(); @@ -5869,9 +5945,15 @@ impl NodeState { .metrics .record_grid_points(output.series.time.len()); self.buffer.metrics.record_output(1, valid_values(&output)); + self.track(&output)?; let retained = window.retained_bytes(); - self.buffer - .resize(retained.saturating_add(held).saturating_add(queued_bytes))?; + let emitted = self.emitted_bytes(); + self.buffer.resize( + retained + .saturating_add(emitted) + .saturating_add(held) + .saturating_add(queued_bytes), + )?; let bytes = evaluated_bytes(&output); self.enqueue_output(output)?; queued_bytes = queued_bytes.saturating_add(bytes); @@ -5909,6 +5991,26 @@ impl NodeState { } Ok(()) } + + /// Records what this node hands out under its output label set. + /// + /// A node that rewrites labels is where two input series become indistinguishable, so this is where a + /// second one is refused: checking at the root would be too late for a window above, which would + /// already have evaluated the two as two series. + fn track(&mut self, output: &EvaluatedChunk) -> Result<()> { + match self.emitted.as_mut() { + Some(emitted) => emitted.occupy(output), + // A node that keeps its input's labels cannot make two series one. + None => Ok(()), + } + } + + /// The bytes the label sets this node has handed out hold. + /// + /// Zero for a node that keeps its input's labels, which keeps no such table. + fn emitted_bytes(&self) -> usize { + self.emitted.as_ref().map_or(0, Emitted::bytes) + } } impl InputSlot { @@ -6366,6 +6468,24 @@ fn missing_row(index: usize) -> DataFusionError { )) } +/// The rejection of a chunk holding a value where the same field already holds one. +/// +/// No merge resolves this one: at `position`, field `field` of the label set `labels` would carry two +/// values, which is what Prometheus rejects for two series whose label sets became equal. +fn occupied_position( + labels: &LabelSet, + field: usize, + run: &Run, + previous: &Run, +) -> DataFusionError { + DataFusionError::Execution(format!( + "vector cannot contain metrics with the same labelset {}: two series hold a value of field \ + {field} at position {position} ({previous:?} and {run:?})", + crate::grid_series::render_label_set(labels), + position = run.start.max(previous.start), + )) +} + /// The rejection of an input that ended without the chunk closing its series. /// /// A tree with no window has no evaluator to hold the series, so the input node is the one node left that @@ -6499,6 +6619,20 @@ fn broadcast_runtime( }) } +/// Whether the [`Emitted`] table `broadcast` would keep only repeats the one its vector side keeps. +/// +/// Literal arithmetic leaves missing samples missing and floats at their positions, so its output occupancy +/// is a subset of the window's, and the second label drop removes only empty-valued labels that +/// [`label_set`] ignores. Arithmetic therefore cannot create a collision the window did not already refuse. +fn elides_emitted_table(broadcast: &BroadcastRuntime, vector: &NativeNode) -> bool { + matches!(broadcast.scalar, BroadcastScalar::Literal(_)) + && matches!(broadcast.op, BroadcastOp::Arithmetic(_)) + && matches!( + &vector.expr, + NativeExpr::Window { op, .. } if op.function.drops_metric_name() + ) +} + /// The operation a binary node's `matching` describes. /// /// A comparison is a filter unless it was bound as `bool`, and a filter keeps the label set it read while diff --git a/src/promql/src/native/execute/tests.rs b/src/promql/src/native/execute/tests.rs index d61e665e7b8..d16241b3d41 100644 --- a/src/promql/src/native/execute/tests.rs +++ b/src/promql/src/native/execute/tests.rs @@ -15,7 +15,7 @@ //! What the runtime has to get right: each layer's grid, per-node accounting, rejection and //! release. -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Weak}; use datafusion::error::DataFusionError; @@ -43,10 +43,10 @@ fn is_resources_exhausted(error: &DataFusionError) -> bool { } use super::{ - BinaryRuntime, InputId, InputSlot, InstantRuntime, NativeChunk, NativeExecutionState, + BinaryRuntime, Emitted, InputId, InputSlot, InstantRuntime, NativeChunk, NativeExecutionState, NativeExecutor, NodeState, PairKernel, Reader, RetainedInput, RuntimeNode, RuntimeOp, - SourceFacts, buffered_fields, buffered_time, evaluated_bytes, grid_chunk_bytes, grid_end_ms, - grid_position, map_chunk, map_workspace_bytes, raw_bytes, vector_series_with_k, + absorb_rebuilds, buffered_fields, buffered_time, evaluated_bytes, grid_chunk_bytes, + grid_end_ms, grid_position, map_chunk, map_workspace_bytes, raw_bytes, vector_series_with_k, vector_workspace_bytes, }; use crate::evaluated::EvaluatedChunk; @@ -58,7 +58,6 @@ use crate::instant_eval::{instant_candidate_span, select_instants}; use crate::map::MathOp; use crate::map_label::LabelRewrite; use crate::map_time::DateFunction; -use crate::native::canonicalize::merge_label_rewrites; use crate::native::metrics::{NodeMetric, NodeMetrics}; use crate::native::plan::{ AggregateKind, AggregateStage, AggregateStateSchema, BoundAggregate, BoundInput, BoundInstant, @@ -123,93 +122,6 @@ impl MemoryPool for DropOrderPool { } } -/// A pool that records every reservation it is asked to admit, so a test can count the calls that -/// made one rather than infer them from the figure a reservation ends at. -/// -/// The figure cannot tell two reservations of no bytes apart from one, and it cannot tell a -/// reservation made twice from one made once and then left alone at the same size; the calls can. -/// Each entry is the consumer's name and the bytes it asked for, in the order they were asked for, -/// and the largest total the pool was asked to hold at once is kept beside them: that figure is -/// what a limit has to admit for a run to fit, and it is a maximum inside a call, which the -/// reservation's size between calls does not show. -#[derive(Debug)] -struct GrowCallPool { - inner: GreedyMemoryPool, - calls: std::sync::Mutex>, - peak: AtomicUsize, -} - -impl std::fmt::Display for GrowCallPool { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "grow-call test pool") - } -} - -impl GrowCallPool { - fn new(bytes: usize) -> Self { - Self { - inner: GreedyMemoryPool::new(bytes), - calls: std::sync::Mutex::new(Vec::new()), - peak: AtomicUsize::new(0), - } - } - - /// The largest total this pool was asked to hold at one time. - fn peak(&self) -> usize { - self.peak.load(Ordering::SeqCst) - } - - /// The bytes the consumer called `name` asked this pool to admit, in order. - fn grows(&self, name: &str) -> Vec { - self.calls - .lock() - .unwrap() - .iter() - .filter(|(consumer, _)| consumer == name) - .map(|(_, bytes)| *bytes) - .collect() - } - - fn record(&self, reservation: &MemoryReservation, additional: usize) { - self.peak.fetch_max( - reservation.size().saturating_add(additional), - Ordering::SeqCst, - ); - self.calls - .lock() - .unwrap() - .push((reservation.consumer().name().to_string(), additional)); - } -} - -impl MemoryPool for GrowCallPool { - fn name(&self) -> &str { - "grow-call test pool" - } - - fn grow(&self, reservation: &MemoryReservation, additional: usize) { - self.record(reservation, additional); - self.inner.grow(reservation, additional); - } - - fn try_grow( - &self, - reservation: &MemoryReservation, - additional: usize, - ) -> Result<(), DataFusionError> { - self.record(reservation, additional); - self.inner.try_grow(reservation, additional) - } - - fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { - self.inner.shrink(reservation, shrink); - } - - fn reserved(&self) -> usize { - self.inner.reserved() - } -} - fn grid(start: i64, step: i64, len: u64) -> Arc { Arc::new(EvalGrid::try_new(GridSpec::try_new(start, step, len).unwrap()).unwrap()) } @@ -423,7 +335,7 @@ fn instant( fn window_and_limitk_collect_only_unified_contributions() { let grid = grid(0, 1_000, 2); - let build_with = |node: NativeNode, sources: &SourceFacts| { + let build = |node: NativeNode| { let tree = NativeTree::try_new(node).unwrap(); let mut inputs = Vec::new(); let mut binary_before = 0; @@ -434,11 +346,9 @@ fn window_and_limitk_collect_only_unified_contributions() { &mut binary_before, "root".to_string(), None, - sources, ) .unwrap() }; - let build = |node: NativeNode| build_with(node, &SourceFacts::none()); let raw_window = build(window_with( evaluated_input(Arc::clone(&grid)), @@ -553,54 +463,6 @@ fn window_and_limitk_collect_only_unified_contributions() { !vector.merge_input, "limitk must not re-merge a window that already unifies its contributions" ); - - // A window that drops the metadata labels collects its child only when something it drops tells - // the child's series apart. The child here is `abs(x)`, which already dropped the name: a caller - // that cannot say which metric the scan reads collects, and one that says the scan reads a - // single metric whose tags carry neither `__type__` nor `__unit__` does not - the same proof the - // canonical pass used when it decided this window needs no merge above it. - let dropped_name_window = || { - window_with( - window_with( - map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]), - RangeFunction::Rate, - 300_000, - Some(Arc::clone(&grid)), - Arc::clone(&grid), - ), - RangeFunction::LastOverTime, - 300_000, - Some(Arc::clone(&grid)), - Arc::clone(&grid), - ) - }; - let RuntimeOp::Window { - merge: unknown_facts, - .. - } = build(dropped_name_window()).op - else { - panic!("expected window runtime node"); - }; - assert!( - unknown_facts.is_some(), - "a caller that knows nothing about the scan collects" - ); - let named = SourceFacts::uniform( - vec!["host".to_string(), "__name__".to_string()], - Some(Arc::from("some_metric")), - 1, - ); - let RuntimeOp::Window { - merge: proven_facts, - .. - } = build_with(dropped_name_window(), &named).op - else { - panic!("expected window runtime node"); - }; - assert!( - proven_facts.is_none(), - "a single metric's dropped name tells none of its series apart" - ); } #[test] @@ -1391,214 +1253,6 @@ fn eval_time_reports_seconds_and_dates_of_the_grid() { assert_eq!(pool.reserved(), 0); } -/// The bytes a grid's timestamps take, which is what an execution reserves for the array. -fn grid_array_bytes(grid: &EvalGrid) -> usize { - usize::try_from(grid.spec().len).unwrap() * std::mem::size_of::() -} - -/// A grid whose timestamps are still deferred, as a subquery's or an anchored step's is: the plan -/// is compiled without allocating them, and the execution that reads them allocates once. -fn deferred_grid(start: i64, step: i64, len: u64) -> Arc { - let grid = EvalGrid::try_deferred(GridSpec::try_new(start, step, len).unwrap()).unwrap(); - assert!(!grid.is_materialized()); - Arc::new(grid) -} - -/// The node that materializes a deferred grid reserves its array and keeps that reservation. -/// -/// A deferred grid is what keeps compilation proportional to the plan rather than to the timestamp -/// count, so the array it eventually needs is allocated at execution: the node that triggers the -/// allocation reserves it, and a pool that cannot hold it refuses the query instead of letting it -/// take the array unasked. What it reserved stays reserved for as long as the node lives, because -/// the array outlives the chunk the node read it for. -#[test] -fn a_node_that_materializes_a_deferred_grid_reserves_it_for_its_lifetime() { - let points = 1_000u64; - let grid = deferred_grid(0, 60_000, points); - let array = grid_array_bytes(&grid); - let tree = NativeTree::try_new(vector_of(eval_time(None, Arc::clone(&grid)))).unwrap(); - - // One byte short of the array: the node that would allocate it is refused instead. - let pool = bounded_pool(array - 1); - let mut execution = executor(&tree, Arc::clone(&pool)); - let error = execution.next_chunk().unwrap_err(); - assert!(is_resources_exhausted(&error), "{error}"); - assert!( - !grid.is_materialized(), - "a refused node materialized the grid anyway" - ); - assert_eq!(pool.reserved(), 0); - - // Room for the array and the chunk built from it: the query answers, and what the node keeps - // once its chunk has been handed on is the array alone. - let chunk = grid_chunk_bytes(&grid).unwrap(); - let pool = bounded_pool(array + chunk); - let mut execution = executor(&tree, Arc::clone(&pool)); - assert!(execution.next_chunk().unwrap().is_some()); - assert!( - grid.is_materialized(), - "the node did not materialize the grid" - ); - assert_eq!( - execution.state().reserved_bytes(), - array, - "the materialized grid was not kept reserved after its chunk was handed on" - ); - drop(execution); - assert_eq!( - pool.reserved(), - 0, - "the array's reservation outlived its node" - ); -} - -/// A grid that is already materialized is charged to nobody: the execution reads the array that is -/// there, and only the node that allocates it reserves it. -/// -/// This is also what keeps two nodes reading one grid from charging it twice, and what makes a grid -/// the plan built eagerly -- [`EvalGrid::try_new`] rather than [`EvalGrid::try_deferred`] -- cost -/// the execution nothing. -#[test] -fn a_materialized_grid_is_not_reserved_again() { - let points = 1_000u64; - let grid = grid(0, 60_000, points); - let array = grid_array_bytes(&grid); - assert!(grid.is_materialized()); - let chunk = grid_chunk_bytes(&grid).unwrap(); - let tree = NativeTree::try_new(vector_of(eval_time(None, Arc::clone(&grid)))).unwrap(); - // A pool one byte short of the array *and* the chunk: the deferred grid above needed both, and - // this one needs the chunk alone. - let pool = bounded_pool(array + chunk - 1); - let mut execution = executor(&tree, Arc::clone(&pool)); - assert!(execution.next_chunk().unwrap().is_some()); - assert_eq!(execution.state().reserved_bytes(), 0); - drop(execution); - assert_eq!(pool.reserved(), 0); -} - -/// A window reads the grid's timestamps to place every point it decides, so it materializes a -/// deferred output grid and keeps that reservation as well. -#[test] -fn a_window_reserves_the_grid_it_materializes() { - let points = 1_000u64; - let grid = deferred_grid(0, 60_000, points); - let array = grid_array_bytes(&grid); - let schema = raw_schema(); - let tree = NativeTree::try_new(window(input(), Arc::clone(&grid))).unwrap(); - - // A pool short of the array refuses the window instead of letting it take the array unasked. - let short = bounded_pool(array - 1); - let mut execution = executor(&tree, Arc::clone(&short)); - execution - .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], true)) - .unwrap(); - let error = execution.next_chunk().unwrap_err(); - assert!(is_resources_exhausted(&error), "{error}"); - assert!( - !grid.is_materialized(), - "a refused window materialized the grid anyway" - ); - assert_eq!(short.reserved(), 0); - - // With room for the array the window keeps it while it holds the series it is evaluating. - let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool)); - execution - .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) - .unwrap(); - drop(execution.next_chunk().unwrap()); - assert!( - grid.is_materialized(), - "the window did not materialize the grid" - ); - let held = execution.state().reserved_bytes(); - assert!( - held >= array, - "the window gave the grid's array back: {held} bytes held, {array} reserved for the grid" - ); - drop(execution); - assert_eq!(pool.reserved(), 0); -} - -/// A node that reads one grid reserves its timestamps once, however many chunks it is handed on -/// that grid. -/// -/// A collecting window reaches its output grid for every chunk it collects and again when it -/// flushes what it collected, and none of those calls finds the array materialized: the timestamps -/// are read by the flush, so what the calls before it reserve is an array that is still deferred. -/// What keeps the execution from charging one array per chunk is the node's record of the -/// reservation it made, which is what this pool counts, rather than the figure the reservation ends -/// at: a reservation the pool is asked for once is one call, whatever it is worth. -#[test] -fn reserve_grid_reserves_the_array_once_however_often_it_is_called() { - let grid = deferred_grid(0, 60_000, 1_000); - let array = grid_array_bytes(&grid); - let name = "native.window at root"; - let pool = Arc::new(GrowCallPool::new(1 << 20)); - let watched: Arc = Arc::clone(&pool) as Arc; - let mut state = NodeState::new(name, Arc::new(NodeMetrics::default()), &watched); - - // A collecting window's five chunks and the flush after them: six calls, one array. - for _ in 0..6 { - state.buffer.reserve_grid(&grid).unwrap(); - } - assert_eq!( - pool.grows(name), - vec![array], - "the node asked the pool to admit the array more than once" - ); - assert_eq!(state.buffer.reservation.size(), array); - assert!( - !grid.is_materialized(), - "reserving for the array read the timestamps the allocation is deferred for" - ); - - // A re-estimate carries the array on top of what it measured, and reaching the grid again is - // not a second reservation of it. - state.buffer.resize(128).unwrap(); - assert_eq!(state.buffer.reservation.size(), array + 128); - state.buffer.reserve_grid(&grid).unwrap(); - assert_eq!(pool.grows(name), vec![array, 128]); - - drop(state); - assert_eq!( - pool.reserved(), - 0, - "the array outlived the node it was reserved for" - ); -} - -/// A grid the plan already materialized is charged to nobody, however often a node reaches it. -/// -/// This is the other path a node takes through [`NodeBuffer::reserve_grid`](super::NodeBuffer): the -/// timestamps are there already, so the node that reads them allocates nothing and reserves -/// nothing, and what its own re-estimates add is its input alone. -#[test] -fn reserve_grid_charges_nothing_for_a_materialized_grid() { - let grid = grid(0, 60_000, 1_000); - assert!(grid.is_materialized()); - let name = "native.window at root"; - let pool = Arc::new(GrowCallPool::new(1 << 20)); - let watched: Arc = Arc::clone(&pool) as Arc; - let mut state = NodeState::new(name, Arc::new(NodeMetrics::default()), &watched); - - for _ in 0..4 { - state.buffer.reserve_grid(&grid).unwrap(); - } - assert!( - pool.grows(name).is_empty(), - "a node reserved for an array it did not allocate" - ); - assert_eq!(state.buffer.reservation.size(), 0); - - state.buffer.resize(64).unwrap(); - assert_eq!(pool.grows(name), vec![64]); - assert_eq!(state.buffer.reservation.size(), 64); - - drop(state); - assert_eq!(pool.reserved(), 0); -} - /// `vector(c)` hands the constant on as one unlabelled series over the whole grid. #[test] fn vector_of_a_constant_is_one_unlabelled_series() { @@ -3582,8 +3236,8 @@ fn two_series_onto_one_label_set_under_a_parameterized_topk_are_merged() { /// /// The pair below maps into one label set and both hold a value at position 0 of the field, which /// no merge resolves and Prometheus rejects for two series whose label sets became equal. The -/// refusal is the merge's, which the window above runs on the input it collected before it evaluates -/// anything: nothing reaches the evaluator until the pieces of one label set are one series. +/// refusal is the map's own, before the window above evaluates anything, and it names the position +/// rather than a closed label set: a closed source series is not a closed label set. #[test] fn two_series_onto_one_label_set_after_a_map_are_rejected() { let inner_grid = grid(0, 60_000, 2); @@ -3630,8 +3284,6 @@ fn two_series_onto_one_label_set_after_a_map_are_rejected() { ), ) .unwrap(); - // The window collects its input, so the two pieces are merged when the input ends. - execution.end_of_input().unwrap(); let error = match execution.next_chunk() { Ok(None) => panic!("the second series was accepted at a position the first one holds"), Ok(Some(chunk)) => panic!( @@ -3641,14 +3293,14 @@ fn two_series_onto_one_label_set_after_a_map_are_rejected() { Err(error) => error, }; let message = error.to_string(); - assert!( - message.contains("vector cannot contain metrics with the same labelset"), - "the rejection does not name the defect: {message}" - ); assert!( message.contains("job"), "the rejection does not name the label set: {message}" ); + assert!( + message.contains("position 0"), + "the rejection does not name the shared position: {message}" + ); } /// An empty tag and an absent one are two source series and one canonical label set. @@ -3856,9 +3508,10 @@ fn a_merge_flush_reserves_every_queued_output() { assert_eq!(first.series.metric.tags()[0].value.as_ref(), "a"); assert!(first.is_last, "a merged series is handed over closed"); - // What the root holds once one of the two answers left: the answer still queued and the - // evaluator that evaluated it. The borrow ends before the next pull. - let (queued, queued_bytes, retained, held) = { + // What the root holds once one of the two answers left: the answer still queued, the label sets + // the node handed out under, and the evaluator that evaluated them. The borrow ends before the + // next pull. + let (queued, queued_bytes, retained, emitted, held) = { let RuntimeOp::Window { window, state, .. } = &execution.state().live.as_ref().unwrap().root.op else { @@ -3869,6 +3522,7 @@ fn a_merge_flush_reserves_every_queued_output() { state.queued.len(), queued_bytes, window.as_ref().map_or(0, RangeEvaluator::retained_bytes), + state.emitted_bytes(), state.buffer.reservation.size(), ) }; @@ -3879,7 +3533,7 @@ fn a_merge_flush_reserves_every_queued_output() { ); assert_eq!( held, - queued_bytes + retained, + queued_bytes + retained + emitted, "the reservation after the first pull is not what the node still holds" ); assert_eq!(pool.reserved(), execution.state().reserved_bytes()); @@ -4015,173 +3669,6 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() { ); } -/// A `sum_over_time` window over a map over a source: the map can rewrite label sets, so the window -/// collects its whole input before it evaluates any of it. -/// -/// The range covers every point of the `points` the output grid has, so the series the window -/// eventually folds is the whole series it collected. -fn collecting_window_tree(grid: &Arc, points: u64) -> NativeTree { - let step = grid.spec().step_ms; - NativeTree::try_new(window_with( - map(evaluated_input(Arc::clone(grid)), vec![MathOp::Abs]), - RangeFunction::SumOverTime, - step.saturating_mul(points as i64), - Some(Arc::clone(grid)), - Arc::clone(grid), - )) - .unwrap() -} - -/// Delivers one series as `chunks` one-point chunks of `1.0` to a collecting window's source, -/// pulling after each of them. -/// -/// The chunks do not close the series except on the last one, so the window is still collecting -/// when the call returns and nothing has been answered: what changed is only what it keeps of the -/// chunks it was handed. -fn push_one_point_chunks(execution: &mut NativeExecutor, schema: &SeriesSchema, chunks: usize) { - for position in 0..chunks { - execution - .push_evaluated( - InputId::new(0), - evaluated_chunk(schema, position, vec![1.0], position + 1 == chunks), - ) - .unwrap(); - assert!( - execution.next_chunk().unwrap().is_none(), - "the collecting window answered at position {position} before its input ended" - ); - } -} - -/// A collecting window is handed its input one chunk at a time and reaches its output grid on every -/// one of them, so what it reserves for the timestamps must not grow with the chunk count. -/// -/// The window reads the grid's timestamps only when it flushes, so the array is still deferred when -/// the chunks arrive: a reservation made per call would charge the pool one array for every chunk -/// -- `chunks + 1` of them, the last one on the flush -- for the one array the execution holds. The -/// same chain over an eager grid charges nothing at all for the timestamps, because the plan -/// already allocated them, so what the two figures differ by is exactly what the deferred run -/// reserved: one array. -#[test] -fn a_collecting_window_reserves_its_grid_once_for_all_its_chunks() { - let points = 1_000u64; - let chunks = 64usize; - // What the chain holds once it has collected the series, for a grid whose array the plan - // built and for one whose array the flush has to build. - let held_after = |grid: Arc| { - let schema = evaluated_schema(Arc::clone(&grid)); - let tree = collecting_window_tree(&grid, points); - let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool)); - push_one_point_chunks(&mut execution, &schema, chunks); - let held = execution.state().reserved_bytes(); - assert_eq!( - held, - pool.reserved(), - "the state and the pool disagree on what the collected input holds" - ); - let out = execution.finish().expect("the collected series evaluates"); - assert_eq!( - decided(&out) - .last() - .map(|(position, value)| (*position, *value)), - Some((points as usize - 1, Some(chunks as f64))), - "the window did not fold the whole collected series" - ); - assert_eq!(pool.reserved(), 0); - held - }; - - let deferred = deferred_grid(0, 60_000, points); - let array = grid_array_bytes(&deferred); - let eager_held = held_after(grid(0, 60_000, points)); - let deferred_held = held_after(Arc::clone(&deferred)); - assert_eq!( - deferred_held - eager_held, - array, - "the window reserved {} arrays for the one array it read: {deferred_held} bytes held over \ - a deferred grid against {eager_held} over an eager one, {array} per array", - (deferred_held - eager_held) / array - ); - assert!( - deferred.is_materialized(), - "the flush did not read the grid it answered on" - ); -} - -/// A pool sized for what a collecting window holds is not exhausted by the number of chunks it was -/// handed. -/// -/// The chain collects `chunks` one-point chunks of one series over a deferred grid, and the bytes it -/// needs are those fragments, the merge's workspace, the answer and one array of timestamps. The -/// same chain over an eager grid needs all of that and nothing for the timestamps, so its peak -/// plus a few arrays is a budget that holds the deferred run -- and a reservation made per call, -/// one array for each of `chunks + 1` calls, would run out of it after a handful of chunks. A -/// chunk count is then something the query carries rather than something it is sized by. -#[test] -fn many_chunks_do_not_exhaust_a_pool_that_holds_one_grid() { - let points = 1_000u64; - let chunks = 256usize; - let deferred = deferred_grid(0, 60_000, points); - let array = grid_array_bytes(&deferred); - let schema = evaluated_schema(Arc::clone(&deferred)); - - // What the same chain holds once it has collected the series, and the most its pool has to - // admit at one time, over an eager grid: the timestamps are the plan's, so neither figure has - // anything in it for them. - let eager = grid(0, 60_000, points); - let watched = Arc::new(GrowCallPool::new(1 << 20)); - let eager_pool: Arc = Arc::clone(&watched) as Arc; - let tree = collecting_window_tree(&eager, points); - let mut execution = executor(&tree, eager_pool); - push_one_point_chunks( - &mut execution, - &evaluated_schema(Arc::clone(&eager)), - chunks, - ); - let eager_held = execution.state().reserved_bytes(); - let out = execution - .finish() - .expect("the collected series evaluates over an eager grid"); - assert_eq!( - decided(&out) - .last() - .map(|(position, value)| (*position, *value)), - Some((points as usize - 1, Some(chunks as f64))), - "the window did not fold the whole collected series" - ); - let eager_peak = watched.peak(); - drop(execution); - assert_eq!(watched.reserved(), 0); - - // Room for what the eager chain needed and eight arrays: the one the deferred grid holds and - // slack for the chunk arriving. One array for every chunk does not fit in it. - let pool = bounded_pool(eager_peak + 8 * array); - let tree = collecting_window_tree(&deferred, points); - let mut execution = executor(&tree, Arc::clone(&pool)); - push_one_point_chunks(&mut execution, &schema, chunks); - assert_eq!( - execution.state().reserved_bytes(), - eager_held + array, - "the window reserved more than the one array it holds" - ); - assert_eq!(pool.reserved(), eager_held + array); - // The flush reads the timestamps, folds the collected series and hands the answer on, and a - // pool that holds the fragments, the workspace and the one array admits that too. - let out = execution.finish().expect("the collected series evaluates"); - assert_eq!( - decided(&out) - .last() - .map(|(position, value)| (*position, *value)), - Some((points as usize - 1, Some(chunks as f64))) - ); - assert!( - deferred.is_materialized(), - "the flush did not read the grid it answered on" - ); - assert_eq!(pool.reserved(), 0, "the release left bytes reserved"); -} - /// A pool that cannot admit what a collecting chain holds fails the execution and gives everything /// back. /// @@ -4332,99 +3819,93 @@ fn a_pool_that_cannot_admit_the_merge_workspace_releases_what_it_held() { ); } -/// A window that drops empty-valued labels is collected when the scan's key columns can spell the -/// pair, and the merge above it rejects the two series as one label set. +/// A window that drops empty-valued labels cannot hand two series on under one label set either. /// /// `{host="web-1", zone=""}` and `{host="web-1"}` are two series to the scan: an empty tag value is /// a present label, an absent one is not. `rate` drops the metric name and the empty-valued labels, -/// so both become `{host="web-1"}` and each covers the same grid. +/// so both become `{host="web-1"}` and each covers the same grid; answering would hand the whole +/// grid out twice under the same labels and the same timestamps. No map is involved, so the +/// rejection has to be the node's own — a check at the root would be too late, because the window +/// above would already have evaluated the two as two series. /// -/// The one fact that tells the pass to collect the two is the scan's primary key - whether one of -/// its columns can hold `NULL` where another holds `''` - and this fixture states that it cannot -/// prove the pair apart, which is what a nullable tag column says. The window's output is therefore -/// collected, and the merge is where the second value of one label set at one position is refused: -/// Prometheus's answer, rather than two pieces of one series leaving unchecked. -/// -/// The merge is the canonical pass's decision, not this fixture's: the tree is built as the window -/// alone and handed to `merge_label_rewrites` with the facts a planner states - it reads one metric -/// and binds `host`, `zone` and `__name__`, and its key columns can hold that pair - so what is -/// exercised is the pass's answer. +/// The two cover the same positions of the same field, which is the collision no merge resolves: +/// the refusal names the position, not a closed label set, because closing a source series says +/// nothing about the label set it shared with another one. `rate` needs two samples, so neither +/// series decides position 0 -- it holds the single sample at its own timestamp -- and both decide +/// positions 1 to 4. The first position both occupy is therefore 1. #[test] fn two_series_onto_one_label_set_after_a_window_are_rejected() { let out_grid = grid(0, 60_000, 5); - // One tree per execution: a stopped execution keeps its state. - let tree = || { - merge_label_rewrites( - &NativeTree::try_new(window_with( - input(), - RangeFunction::Rate, - 300_000, - None, - Arc::clone(&out_grid), - )) - .unwrap(), - &[vec![ - "host".to_string(), - "zone".to_string(), - "__name__".to_string(), - ]], - &[Some(Arc::::from("requests"))], - // The scan keys its series by a column that can hold `NULL` in one series and `''` in - // another, so its labels alone cannot prove the drop. - false, - ) - .unwrap() - }; - let tree = tree(); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "the key columns cannot tell the pair apart, so the pass collects it: {tree}" - ); + let tree = NativeTree::try_new(window_with( + input(), + RangeFunction::Rate, + 300_000, + None, + Arc::clone(&out_grid), + )) + .unwrap(); let schema = raw_schema(); let counter = [0, 60_000, 120_000, 180_000, 240_000]; - // The second series is a different one to the scan and the same one to `rate`. let mut execution = executor(&tree, pool()); - for (key, metric, values) in [ - ( - b"series-a".as_slice(), - host_metric(Some("")), - [1.0, 2.0, 3.0, 4.0, 5.0], + execution + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-a", + host_metric(Some("")), + &counter, + &[1.0, 2.0, 3.0, 4.0, 5.0], + true, + ), + ) + .unwrap(); + let first = execution + .next_chunk() + .unwrap() + .expect("the first series settles the grid"); + // The decided positions of the first series, which are the ones the second one has to share. + assert_eq!( + first.series.time.runs().count(), + 1, + "the first series decided more than one run" + ); + assert_eq!( + first.series.time.first_position(), + Some(1), + "`rate` decided a position the single sample at 0 cannot settle" + ); + assert!(first.is_last); + + // The second series is a different one to the scan and the same one to `rate`. + execution + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-b", + host_metric(None), + &counter, + &[6.0, 7.0, 8.0, 9.0, 10.0], + true, + ), + ) + .unwrap(); + let error = match execution.next_chunk() { + Ok(None) => panic!("the second series settled nothing at all"), + Ok(Some(chunk)) => panic!( + "a second series left the window under the first one's label set: {:?}", + decided(&[chunk]) ), - ( - b"series-b".as_slice(), - host_metric(None), - [6.0, 7.0, 8.0, 9.0, 10.0], - ), - ] { - execution - .push_raw( - InputId::new(0), - raw_chunk_of(&schema, key, metric, &counter, &values, true), - ) - .unwrap(); - } - execution.end_of_input().unwrap(); - let mut rejected = None; - loop { - match execution.next_chunk() { - Ok(None) => break, - // The first piece may settle before the second collides with it; what must not happen is - // both leaving, and neither is the answer the merge owes this pair. - Ok(Some(_)) => {} - Err(error) => { - rejected = Some(error); - break; - } - } - } - let error = rejected.unwrap_or_else(|| { - panic!("the merge handed two pieces of one label set over instead of refusing the pair") - }); + Err(error) => error, + }; let message = error.to_string(); + assert!(message.contains("host"), "{message}"); + assert!(message.contains("web-1"), "{message}"); assert!( - message.contains("vector cannot contain metrics with the same labelset"), - "the rejection does not name the shared label set: {message}" + message.contains("position 1"), + "the rejection does not name the first position both series hold: {message}" ); // Two series carrying different hosts stay distinct, which is what a rate over a label that @@ -4467,271 +3948,18 @@ fn two_series_onto_one_label_set_after_a_window_are_rejected() { assert_eq!(out.len(), 2, "two series over two hosts are two series"); } -/// A name-dropping node over one metric is proved by the labels alone only when the scan's key -/// columns always decode a value; a scan whose keys can spell `NULL` and `''` is collected, and the -/// merge above the drop is what rejects the pair. -/// -/// `requests{host="web-1", zone=""}` and `requests{host="web-1"}` are two series to a scan and one -/// label set to everything that drops empty values: `timestamp(x)`, `rate(x[5m])` and -/// `timestamp(rate(x[5m]))` all drop the metric name and the empty `zone` with it. The name an -/// individual series carries is the one the whole scan carries, so the labels alone prove the drop -/// of that name - but not of the empty value: a primary-key column that holds `NULL` in one series -/// and `''` in another is exactly this pair, and the pass collects it when the scan states that its -/// keys cannot tell them apart. -/// -/// Without that fact the drop is proved and nothing is collected, which is what the last block pins: -/// the two hosts below are two series to the scan and two to every kernel above it. -/// -/// The statements collected for the key columns alone are counted, not silently paid for: see -/// `duplicate_labelset_check_skipped` in the query planner. -#[test] -fn a_name_dropping_node_over_a_nullable_key_is_collected() { - let out_grid = grid(0, 60_000, 5); - let schema = raw_schema(); - let counter = [0, 60_000, 120_000, 180_000, 240_000]; - let tags = vec![vec![ - "host".to_string(), - "zone".to_string(), - "__name__".to_string(), - ]]; - let metrics = [Some(Arc::::from("requests"))]; - // `requests{host=host}`, the shape the proved case is told apart by. - fn requests(host: &str) -> SeriesLabels { - SeriesLabels::try_new( - Some("requests".into()), - vec![Label { - name: "host".into(), - value: host.into(), - }], - ) - .unwrap() - } - // What the merge refuses, or `None` when every piece left it. - fn rejected(execution: &mut NativeExecutor) -> Option { - loop { - match execution.next_chunk() { - Ok(None) => return None, - // A piece may settle before a later one collides with it. - Ok(Some(_)) => {} - Err(error) => return Some(error.to_string()), - } - } - } - - let stamps = || { - let mut node = instant(input(), 300_000, 0, Arc::clone(&out_grid)); - let NativeExpr::Instant { op, .. } = &mut node.expr else { - unreachable!() - }; - op.value = InstantValue::Timestamp; - node - }; - let rate = || { - window_with( - input(), - RangeFunction::Rate, - 300_000, - None, - Arc::clone(&out_grid), - ) - }; - let evaluated_stamps = || { - let inner = rate(); - let output = inner.output.clone(); - NativeNode { - expr: NativeExpr::Timestamp { - input: Box::new(inner), - }, - output, - } - }; - let passed = |root: NativeNode, keys_survive: bool| { - let tree = NativeTree::try_new(root).unwrap(); - merge_label_rewrites(&tree, &tags, &metrics, keys_survive) - .expect("the pass produces a legal tree") - }; - let push = |execution: &mut NativeExecutor, second: SeriesLabels| { - execution - .push_raw( - InputId::new(0), - raw_chunk_of( - &schema, - b"series-a", - host_metric(Some("")), - &counter, - &[1.0, 2.0, 3.0, 4.0, 5.0], - true, - ), - ) - .unwrap(); - execution - .push_raw( - InputId::new(0), - raw_chunk_of( - &schema, - b"series-b", - second, - &counter, - &[6.0, 7.0, 8.0, 9.0, 10.0], - true, - ), - ) - .unwrap(); - }; - - // A key column that can hold `NULL` in one series and `''` in another: the drop is collected, - // and the merge is where the two pieces of one label set are refused. - let refused = |name: &str, root: NativeNode| { - let tree = passed(root, false); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "`{name}` over a nullable key is collected: {tree}" - ); - let mut execution = executor(&tree, pool()); - push(&mut execution, host_metric(None)); - execution.end_of_input().unwrap(); - let message = rejected(&mut execution) - .unwrap_or_else(|| panic!("`{name}` handed two pieces of one label set over")); - assert!( - message.contains("vector cannot contain metrics with the same labelset"), - "`{name}`: the rejection does not name the shared label set: {message}" - ); - }; - refused("timestamp", stamps()); - refused("rate", rate()); - refused("timestamp(rate)", evaluated_stamps()); - - // Two series a non-empty label tells apart stay two. Over keys that cannot spell the pair the - // drop is proved and nothing is collected - an ordinary `abs`/`rate`/`timestamp` over a normal - // metric table - and over a nullable key it is collected, which costs the merge but answers the - // same two series. - let kept = |name: &str, root: NativeNode| { - let tree = passed(root, true); - assert!( - !matches!(tree.root().expr, NativeExpr::Merge { .. }), - "`{name}` keeps every partition: {tree}" - ); - let mut execution = executor(&tree, pool()); - push(&mut execution, requests("web-2")); - let out = run(&mut execution); - assert_eq!(2, out.len(), "`{name}`: two series over two hosts are two"); - assert_ne!( - out[0].series.metric, out[1].series.metric, - "`{name}`: the two hosts were not two label sets" - ); - }; - kept("timestamp", stamps()); - kept("rate", rate()); - kept("timestamp(rate)", evaluated_stamps()); - let kept_over_a_nullable_key = |name: &str, root: NativeNode| { - let tree = passed(root, false); - assert!( - matches!(tree.root().expr, NativeExpr::Merge { .. }), - "`{name}` over a nullable key is collected: {tree}" - ); - let mut execution = executor(&tree, pool()); - push(&mut execution, requests("web-2")); - let out = run(&mut execution); - assert_eq!( - 2, - out.len(), - "`{name}`: two series over two hosts are two, collected or not" - ); - assert_ne!( - out[0].series.metric, out[1].series.metric, - "`{name}`: the two hosts were not two label sets" - ); - }; - kept_over_a_nullable_key("timestamp", stamps()); - kept_over_a_nullable_key("rate", rate()); - kept_over_a_nullable_key("timestamp(rate)", evaluated_stamps()); -} - -/// Two series a `timestamp` hands out under one label set are refused by the merge above it. -/// -/// `timestamp` reports a sample's own time under the metric name dropped, so `a{job="x"}` and -/// `b{job="x"}` are one label set the moment they leave it, whatever their input held. The merge -/// collecting its output is where the second value of one position is reported, as Prometheus -/// refuses two series of one label set; a single series passes through it unchanged. -#[test] -fn two_series_onto_one_label_set_after_a_timestamp_are_rejected() { - let grid = grid(0, 60_000, 2); - let schema = evaluated_schema(Arc::clone(&grid)); - let tree = || { - let input = evaluated_input(Arc::clone(&grid)); - let output = input.output.clone(); - NativeTree::try_new(merge(NativeNode { - expr: NativeExpr::Timestamp { - input: Box::new(input), - }, - output, - })) - .unwrap() - }; - - // One series: the merge groups the one label set `timestamp` leaves it and answers it. - let mut single = executor(&tree(), pool()); - single - .push_evaluated( - InputId::new(0), - evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(1.0), Some(2.0)], - true, - ), - ) - .unwrap(); - let out = run(&mut single); - assert_eq!(out.len(), 1, "one series is one answer"); - let mut expected = named_metric("a", "x"); - expected.remove("__name__"); - assert_eq!(out[0].series.metric, expected); - - // Two series: both leave `timestamp` under `{job="x"}` and both hold a value at position 0. - let mut execution = executor(&tree(), pool()); - for (name, values) in [ - ("a", vec![Some(1.0), Some(2.0)]), - ("b", vec![Some(3.0), Some(4.0)]), - ] { - execution - .push_evaluated( - InputId::new(0), - evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), - ) - .unwrap(); - } - execution.end_of_input().unwrap(); - let error = match execution.next_chunk() { - Ok(None) => panic!("the timestamp of two series settled nothing at all"), - Ok(Some(chunk)) => panic!( - "a second series left `timestamp` under the first one's label set: {:?}", - decided(&[chunk]) - ), - Err(error) => error, - }; - let message = error.to_string(); - assert!( - message.contains("vector cannot contain metrics with the same labelset") - && message.contains("job=\"x\""), - "the rejection does not name the label set the two now share: {message}" - ); -} - /// A chunk that holds a value where the same field already holds one is rejected. /// -/// Each piece is a whole source series that closes itself, and the refusal is about the positions -/// the two hold, not about the close: they are one label set to the merge, and both hold a value of -/// field 0 at position 0. `v0` would carry two values at position 0 under one label set, which no merge can -/// resolve, so the merge collecting the map's output refuses rather than picking one. +/// Neither piece closes its series, and neither needs to: the refusal is about the positions +/// themselves. `v0` would carry two values at position 0 under one label set, which no merge can +/// resolve, so the node refuses rather than picking one. #[test] fn two_series_holding_a_value_at_one_position_are_rejected() { let map_grid = grid(0, 60_000, 2); - let tree = NativeTree::try_new(merge(map( + let tree = NativeTree::try_new(map( evaluated_input(Arc::clone(&map_grid)), vec![MathOp::Abs], - ))) + )) .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); @@ -4743,12 +3971,17 @@ fn two_series_holding_a_value_at_one_position_are_rejected() { execution .push_evaluated( InputId::new(0), - evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, false), ) .unwrap(); } - // The merge holds both pieces until its input ends, and then reports the shared position. - execution.end_of_input().unwrap(); + // The first piece is fine on its own: nothing has occupied any position yet. + let first = execution + .next_chunk() + .unwrap() + .expect("the first piece is handed on as it arrives"); + assert_eq!(sample_values(&first), vec![Some(1.0), Some(2.0)]); + // The second one puts a second value of the same field at position 0. let error = match execution.next_chunk() { Ok(None) => panic!("the map produced no chunk at all"), Ok(Some(chunk)) => panic!( @@ -4758,11 +3991,9 @@ fn two_series_holding_a_value_at_one_position_are_rejected() { Err(error) => error, }; let message = error.to_string(); - assert!( - message.contains("vector cannot contain metrics with the same labelset"), - "{message}" - ); assert!(message.contains("job"), "{message}"); + assert!(message.contains("field 0"), "{message}"); + assert!(message.contains("position 0"), "{message}"); } /// Two closed series under one label set are two pieces of one logical series. @@ -4892,203 +4123,6 @@ fn merge_canonicalizes_complementary_series_and_flushes_only_at_eof() { assert!(execution.next_chunk().unwrap().is_none()); } -/// A table of `wide`- and `wider`-bucket histograms, so two samples of one field differ in payload. -/// -/// The payload is what a merge can lose and still answer the right positions, so every test below -/// names the histogram it expects rather than only counting rows. -fn two_sized_histograms() -> (Arc, u32, u32) { - let mut builder = HistogramBatchBuilder::new(); - let wide = push_histogram(&mut builder, 2); - let wider = push_histogram(&mut builder, 5); - (builder.finish().unwrap(), wide, wider) -} - -/// The histograms field 0 of `chunk` holds, in grid order: `(position, count, buckets)`. -fn histogram_rows(chunk: &EvaluatedChunk) -> Vec<(usize, f64, usize)> { - let field = chunk - .series - .fields - .get(0) - .expect("the chunk carries one field"); - chunk - .series - .time - .positions() - .map(|(position, row)| match field.sample(row) { - SampleRef::Histogram(view) => ( - position, - view.count, - view.positive.buckets.len() + view.negative.buckets.len(), - ), - SampleRef::Missing => panic!("position {position} of the answer holds no sample"), - SampleRef::Float(value) => { - panic!("position {position} of the answer is the float {value}") - } - }) - .collect() -} - -/// Native histogram frames merge by position across the canonical merge, payload and all. -/// -/// This is the merge a metadata drop over a nullable key pays for: one label set arrives as two -/// pieces from the same partition, each holding some of its positions, and what comes out is one -/// series with one histogram per position, each still carrying the payload it was pushed with. A -/// position neither piece holds stays a hole, and the merge owes nothing until its input ends - the -/// closing chunk it answers with is the merged series. -#[test] -fn merge_canonicalizes_complementary_histogram_pieces_and_flushes_only_at_eof() { - let grid = grid(0, 1, 4); - let tree = NativeTree::try_new(merge(evaluated_input(Arc::clone(&grid)))).unwrap(); - let schema = evaluated_schema(Arc::clone(&grid)); - let label = named_metric("same", "x"); - let (table, wide, wider) = two_sized_histograms(); - let mut execution = executor(&tree, pool()); - execution - .push_evaluated( - InputId::new(0), - evaluated_histogram_chunk_of( - &schema, - label.clone(), - 0, - vec![Some(wide), None, None], - &table, - true, - ), - ) - .unwrap(); - execution - .push_evaluated( - InputId::new(0), - evaluated_histogram_chunk_of(&schema, label, 1, vec![None, Some(wider)], &table, true), - ) - .unwrap(); - assert!( - execution.next_chunk().unwrap().is_none(), - "merge exposed output before EOF" - ); - execution.end_of_input().unwrap(); - let output = execution.next_chunk().unwrap().unwrap(); - assert!(output.is_last); - // Position 0 is the first piece's histogram, position 2 the second piece's, and position 1 - a - // hole in both - stays a hole rather than becoming a row. - assert_eq!( - vec![(0, 2.0, 2), (2, 5.0, 5)], - histogram_rows(&output), - "the merged histograms are not the ones the pieces held" - ); - assert!(execution.next_chunk().unwrap().is_none()); -} - -/// Two pieces of one label set holding a histogram at one position are refused. -/// -/// The merge collects the pieces of one label set into one series, so a position two pieces fill -/// would carry two values of one field. That is the collision Prometheus reports, and the histogram -/// payload changes nothing about it: the check reads whether a sample is there, not what it holds. -#[test] -fn a_histogram_merge_rejects_a_position_two_pieces_hold() { - let grid = grid(0, 1, 4); - let tree = NativeTree::try_new(merge(evaluated_input(Arc::clone(&grid)))).unwrap(); - let schema = evaluated_schema(Arc::clone(&grid)); - let label = named_metric("same", "x"); - let (table, wide, wider) = two_sized_histograms(); - let mut execution = executor(&tree, pool()); - for (start, index) in [(0, wide), (0, wider)] { - execution - .push_evaluated( - InputId::new(0), - evaluated_histogram_chunk_of( - &schema, - label.clone(), - start, - vec![None, Some(index)], - &table, - true, - ), - ) - .unwrap(); - } - execution.end_of_input().unwrap(); - let error = match execution.next_chunk() { - Ok(None) => panic!("the merge settled nothing at all"), - Ok(Some(chunk)) => panic!( - "a histogram position two pieces held was handed on: {:?}", - histogram_rows(&chunk) - ), - Err(error) => error, - }; - let message = error.to_string(); - assert!( - message.contains("vector cannot contain metrics with the same labelset"), - "the rejection does not name the shared label set: {message}" - ); -} - -/// A float-producing histogram function above the canonical merge reduces what the merge collected. -/// -/// `histogram_count(rate(h[5m]))` is this shape: the window below it answers histograms, the drop it -/// makes over a nullable key is collected into a merge carrying those frames, and the map above the -/// merge answers one float per histogram it reads. The merge is what proves the frames crossed the -/// boundary intact - a count of a lost payload would not be the count the piece was pushed with - and -/// the map above it drops the positions holding nothing. -#[test] -fn a_float_producing_histogram_map_reduces_the_histograms_a_merge_collected() { - let grid = grid(0, 1, 4); - let tree = NativeTree::try_new(map_ops( - merge(evaluated_input(Arc::clone(&grid))), - vec![MapOp::Histogram(NativeHistogramFunction::Count)], - )) - .unwrap(); - let schema = evaluated_schema(Arc::clone(&grid)); - let label = named_metric("same", "x"); - let (table, wide, wider) = two_sized_histograms(); - let mut execution = executor(&tree, pool()); - execution - .push_evaluated( - InputId::new(0), - evaluated_histogram_chunk_of( - &schema, - label.clone(), - 0, - vec![Some(wide), None, None], - &table, - true, - ), - ) - .unwrap(); - execution - .push_evaluated( - InputId::new(0), - evaluated_histogram_chunk_of( - &schema, - label, - 1, - vec![None, Some(wider), None], - &table, - true, - ), - ) - .unwrap(); - // Both pieces close themselves, so the map's answer comes out of `run` without an explicit end - // of input; the merge still owes nothing until the second piece arrives. - let chunks = run(&mut execution); - let [chunk] = chunks.as_slice() else { - panic!("expected one chunk, got {}", chunks.len()); - }; - // The map answers one float per merged histogram, at the positions the merge kept, and the name - // the map drops is the one the pieces shared. - assert_eq!(chunk.series.metric.name(), None); - assert_eq!( - chunk - .series - .time - .positions() - .map(|(position, _)| position) - .collect::>(), - vec![0, 2] - ); - assert_eq!(sample_values(chunk), vec![Some(2.0), Some(5.0)]); -} - /// A parent reads a merge's answer only once the merge has flushed. /// /// `Merge` is the only node that answers nothing until its input ends, so a parent that treats it @@ -5520,16 +4554,21 @@ fn merge_rejects_overlapping_occupancy() { ); } -/// Two open chunks that share a label set are handed on as they arrive. +/// Two open chunks that share a label set are accepted where no position of a field collides. /// /// `a{job="x"}` and `b{job="x"}` are two series to the input and one label set to `abs`, which /// drops the name. Neither piece says it is the last chunk of its series, so the second one arrives /// while the first is still open. The two occupy disjoint positions of field 0 — a value at 0 and a -/// value at 3 — so nothing about their labels is decided here: a map passes each chunk through as it -/// arrives, and combining two pieces of one label set is -/// [`crate::grid_series::merge_same_labelset`]'s job, which the merge a consumer reads them through -/// performs. A pair holding a value at the same position is what that merge refuses, not this -/// layer. +/// value at 3 — so the label-set check hands both on. +/// +/// The ordering is contiguous because that is the only ordering this runtime executes: the stream +/// below is not one a scan produces, since it never closes `a`, and what is checked here is the +/// occupancy the check reads rather than a delivery order. +/// +/// This is acceptance by that check, and it is not a merge: this layer passes every chunk through +/// as it arrives and keeps no sample state that would combine two series. Merging two series under +/// one label set is [`crate::grid_series::merge_same_labelset`]'s job, and nothing here performs +/// it. #[test] fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides() { let map_grid = grid(0, 60_000, 4); @@ -5588,19 +4627,18 @@ fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides /// A `NaN` is a value and a missing sample is not, so only the `NaN` occupies its position. /// -/// `abs` drops the metric name, so the two series below are one label set to the map, and the merge -/// above it is where the two are read against each other. A position holding no sample is a -/// position the series never claimed, and another series may fill it; a position holding a `NaN` is -/// decided and holds a value, like any other, so a second value there is the collision no merge -/// resolves. +/// `abs` drops the metric name, so the two series below are one label set to the map, which is where +/// the two are read against each other. A position holding no sample is a position the series never +/// claimed, and another series may fill it; a position holding a `NaN` is decided and holds a value, +/// like any other, so a second value there is the collision no merge resolves. #[test] fn a_nan_occupies_its_position_where_a_missing_sample_does_not() { let map_grid = grid(0, 60_000, 1); let tree = || { - NativeTree::try_new(merge(map( + NativeTree::try_new(map( evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Contiguous), vec![MathOp::Abs], - ))) + )) .unwrap() }; let schema = evaluated_schema(Arc::clone(&map_grid)); @@ -5614,24 +4652,28 @@ fn a_nan_occupies_its_position_where_a_missing_sample_does_not() { (vec![Some(1.0)], vec![None], true), (vec![None], vec![Some(f64::NAN)], true), ]; - // Each chunk is a whole source series that closes itself, so the merge reads the two as two - // pieces of one label set rather than as one series' runs. It holds both until the input ends - // and only then answers or refuses the pair. + // Both chunks are open and neither needs to close: what the check reads is which positions a + // chunk occupies, and that does not depend on `is_last`. + let drain = |execution: &mut NativeExecutor| -> Result<(), DataFusionError> { + while execution.next_chunk()?.is_some() {} + Ok(()) + }; for (first, second, accepted) in cases { let mut execution = executor(&tree(), pool()); execution .push_evaluated( InputId::new(0), - evaluated_chunk_of(&schema, named_metric("a", "x"), 0, first.clone(), true), + evaluated_chunk_of(&schema, named_metric("a", "x"), 0, first.clone(), false), ) .unwrap(); + drain(&mut execution).expect("the first open chunk is handed on"); execution .push_evaluated( InputId::new(0), - evaluated_chunk_of(&schema, named_metric("b", "x"), 0, second.clone(), true), + evaluated_chunk_of(&schema, named_metric("b", "x"), 0, second.clone(), false), ) .unwrap(); - let outcome = execution.finish().map(|_| ()); + let outcome = drain(&mut execution); assert_eq!( outcome.is_ok(), accepted, @@ -5817,6 +4859,63 @@ fn a_window_retains_the_samples_of_every_field() { ); } +/// A chunk that produces nothing still leaves the table this node emitted into in its hands. +/// +/// The first chunk decided both grid points and the window handed them out, so the table holds a +/// label set and its runs. The second chunk decides nothing — every point was already decided — so +/// the window yields no chunk, but it has given none of that history back: the reservation has to +/// keep covering the table, not drop it. +#[test] +fn a_chunk_with_no_output_keeps_charging_the_emitted_table() { + let tree = NativeTree::try_new(window_with( + input(), + RangeFunction::LastOverTime, + 1, + None, + grid(0, 60_000, 2), + )) + .unwrap(); + let schema = raw_schema(); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + + execution + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false), + ) + .unwrap(); + let first = execution + .next_chunk() + .unwrap() + .expect("the first chunk decided both points"); + assert_eq!(sample_values(&first), vec![Some(1.0), Some(2.0)]); + drop(first); + let after_output = execution.state().reserved_bytes(); + assert!( + after_output > 0, + "the window handed out a position, which it has to hold" + ); + assert_eq!(after_output, pool.reserved()); + + // Both points are decided, so this chunk decides nothing and the window yields no chunk. + execution + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[120_000], &[3.0], false), + ) + .unwrap(); + assert!( + execution.next_chunk().unwrap().is_none(), + "a chunk after every point was decided produced a chunk" + ); + assert!( + execution.state().reserved_bytes() > 0, + "a chunk with no output released the bytes the emitted table still holds" + ); + assert_eq!(execution.state().reserved_bytes(), pool.reserved()); +} + /// A window cannot run over an interleaved input, and the runtime says so before reading a chunk. /// /// The binding refuses a window whose own input declares the ordering, and the runtime refuses the @@ -6507,6 +5606,195 @@ fn dropping_a_node_drops_its_queue() { assert!(gauge(&metrics, "native.p0.source.peak_reserved_bytes") > 0); } +/// A chunk whose axis and fields disagree is refused, not read past the end of a field. +/// +/// [`Series`] keeps its fields public, so a chunk built by struct literal -- by a caller of this +/// crate as much as by a mistake here -- can carry fewer samples than the axis places rows. The +/// label-set ledger reads every row the axis names, so it is where such a chunk would be read out of +/// bounds; it reports the misalignment against the node that was handed it instead. +#[test] +fn a_series_whose_axis_and_fields_disagree_is_rejected() { + let grid = grid(0, 60_000, 4); + // Two samples placed on a four-row axis: the third row the axis names has nothing to read. + let samples = Samples::from_float(Float64Array::from(vec![1.0, 2.0])); + let time = TimeAxis::Grid(GridRange::try_new(Arc::clone(&grid), 0, 4).unwrap()); + let series = Series { + metric: named_metric("requests", "job"), + time, + fields: SeriesFields::try_from_vec(vec![samples]).unwrap(), + }; + let chunk = EvaluatedChunk::try_new(series, false).expect("the axis is addressed on a grid"); + + let mut emitted = Emitted::default(); + let error = emitted + .occupy(&chunk) + .expect_err("a field with fewer samples than the axis places rows was accepted"); + let message = error.to_string(); + assert!( + message.contains("misaligned") + && message.contains("4 rows") + && message.contains("2 samples"), + "the rejection does not name what the axis places against what the field holds: {message}" + ); +} + +/// A sparse series that only ever moves forward is appended to its history, not merged into it again +/// chunk after chunk. +/// +/// One legal series arrives as eight chunks of one position each, every position a hole apart from +/// the last: no label set is reached twice, no position of a field is occupied twice, and this is +/// what a scan produces as its positions advance. The stored runs still grow by one per chunk, so +/// rebuilding the array after each chunk moves `K(K+1)/2` runs to hold `K` rows of real data, which +/// is the cost this asserts away. +/// +/// The chunks go through [`super::Emitted::occupy`], the helper the runtime records an output with, +/// so what is checked is the history that helper keeps. Three shapes are pushed: chunks landing past +/// every stored run, which have to be appended to; a chunk starting where the last stored run ends, +/// which has to be joined to it rather than left beside it; and chunks landing before what is +/// stored, which cannot be appended to and have to go through the full merge while still leaving the +/// history sorted and disjoint. +#[test] +fn a_sparse_series_that_moves_forward_is_appended_to_its_history() { + let map_grid = grid(0, 60_000, 21); + let schema = evaluated_schema(Arc::clone(&map_grid)); + let mut emitted = Emitted::default(); + + // One row at `position`, which is one run of one position: the same series every time, since a + // map dropping the name leaves them all one label set. + let push = |emitted: &mut Emitted, position: usize| { + let chunk = sparse_chunk( + &schema, + named_metric("a", "x"), + &[(position, 1)], + vec![-1.0], + false, + ); + emitted + .occupy(&chunk) + .unwrap_or_else(|error| panic!("position {position} was refused: {error}")); + }; + // The runs stored under the one label set, which is what `absorb` keeps in position order. + let stored = |emitted: &Emitted| -> Vec<(usize, usize)> { + emitted + .sets + .values() + .next() + .expect("the chunk was recorded under one label set") + .fields + .first() + .expect("the series has one field") + .iter() + .map(|run| (run.start, run.end)) + .collect() + }; + let ordered = |runs: &[(usize, usize)]| { + runs.iter().all(|(start, end)| start < end) + && runs.windows(2).all(|pair| pair[0].1 < pair[1].0) + }; + + // Eight chunks, one position each, a hole between them. + let positions = [4, 6, 8, 10, 12, 14, 16, 18]; + let before = absorb_rebuilds(); + for position in positions { + push(&mut emitted, position); + } + assert_eq!( + stored(&emitted), + vec![ + (4, 5), + (6, 7), + (8, 9), + (10, 11), + (12, 13), + (14, 15), + (16, 17), + (18, 19), + ], + "a sparse series was not stored as the one run each of its chunks occupies" + ); + assert_eq!( + absorb_rebuilds() - before, + 0, + "the {} chunks of a series moving forward rebuilt its history instead of appending to it", + positions.len() + ); + + // A chunk that starts where the last stored run ends touches it, and the two are one run: the + // fast path appends, and the boundary is the part it has to join. + let before = absorb_rebuilds(); + push(&mut emitted, 19); + assert_eq!( + stored(&emitted), + vec![ + (4, 5), + (6, 7), + (8, 9), + (10, 11), + (12, 13), + (14, 15), + (16, 17), + (18, 20), + ], + "a chunk touching the last stored run was left beside it rather than joined to it" + ); + assert_eq!( + absorb_rebuilds() - before, + 0, + "a chunk touching the last stored run was merged rather than appended to it" + ); + + // A chunk landing in a hole before the last stored run cannot be appended to: it joins the runs + // on both sides of it, so it goes through the full merge, which has to join them all the same. + let before = absorb_rebuilds(); + push(&mut emitted, 5); + assert_eq!( + stored(&emitted), + vec![ + (4, 7), + (8, 9), + (10, 11), + (12, 13), + (14, 15), + (16, 17), + (18, 20), + ], + "a chunk in a hole did not join the runs on either side of it" + ); + assert_eq!( + absorb_rebuilds() - before, + 1, + "a chunk landing before the last stored run was appended rather than merged" + ); + + // A chunk before every stored run is the other shape the fast path cannot take. + let before = absorb_rebuilds(); + push(&mut emitted, 1); + let runs = stored(&emitted); + assert_eq!( + runs, + vec![ + (1, 2), + (4, 7), + (8, 9), + (10, 11), + (12, 13), + (14, 15), + (16, 17), + (18, 20), + ], + "a chunk before every stored run did not land in front of them" + ); + assert_eq!( + absorb_rebuilds() - before, + 1, + "a chunk landing before every stored run was appended rather than merged" + ); + assert!( + ordered(&runs), + "the merged history is not sorted, disjoint and joined: {runs:?}" + ); +} + /// A bare selector stamps the last sample at or before each grid point onto its own grid. /// /// The chunk is addressed on the node's output grid, not on the source's timestamps; a grid point @@ -6822,9 +6110,9 @@ fn a_selected_histogram_keeps_its_start_timestamp() { /// Two series that differ only in an empty-valued label are two series, and both are answered. /// -/// The label set a merge groups by drops empty values, so the two would be one set to a merge. A -/// selection keeps the labels it read, empty values included, so it hands the two on apart and -/// collects nothing. +/// The label set the runtime's occupancy table compares drops empty values, so a node that recorded +/// what it handed out under one would refuse the second series. A selection keeps the labels it +/// read, empty values included, so it keeps them apart and records nothing. #[test] fn two_series_differing_only_in_an_empty_label_are_both_answered() { let out_grid = grid(0, 60_000, 1); @@ -10924,6 +10212,41 @@ fn name_of(chunk: &EvaluatedChunk) -> Option<&str> { chunk.series.metric.name() } +/// The [`Emitted`] tables a tree built from `node` keeps, as `(broadcast, window it reads)`. +/// +/// The second answer is `false` when the broadcast's vector side is not a window, which is what +/// tells a table its own node keeps from one the node below it already keeps. +fn emitted_tables(node: NativeNode) -> (bool, bool) { + let tree = NativeTree::try_new(node).unwrap(); + let mut inputs = Vec::new(); + let mut binary_before = 0; + let root = RuntimeNode::build( + tree.root(), + &pool(), + &mut inputs, + &mut binary_before, + "root".to_string(), + None, + ) + .unwrap(); + let RuntimeOp::Broadcast { + left, right, state, .. + } = root.op + else { + panic!("the tree did not build a broadcast at its root"); + }; + let vector = if matches!(left.op, RuntimeOp::Window { .. }) { + left + } else { + right + }; + let window = match vector.op { + RuntimeOp::Window { state, .. } => state.emitted.is_some(), + _ => false, + }; + (state.emitted.is_some(), window) +} + /// A constant on the right multiplies every value of the vector and drops the metric name. /// /// The chunk arrives as it is pushed, before the input ends: a broadcast answers a chunk from that @@ -12097,20 +11420,16 @@ fn a_broadcast_above_a_collecting_window_waits_for_its_flush() { } /// A broadcast rejects distinct input names that become one output label set at occupied points. -/// -/// The multiplication drops the metric name, so `requests` and `requests_total` hold one label set -/// above it and both hold a value at the same positions. The merge above the broadcast is what -/// refuses the pair: the node itself is point-wise and hands every chunk over as it arrives. #[test] fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reservation() { let grid = grid(0, 60_000, 2); let schema = evaluated_schema(Arc::clone(&grid)); - let tree = NativeTree::try_new(merge(broadcast_right( + let tree = NativeTree::try_new(broadcast_right( evaluated_input(Arc::clone(&grid)), 100.0, BinaryOp::Mul, false, - ))) + )) .unwrap(); let pool = pool(); let mut execution = executor(&tree, Arc::clone(&pool)); @@ -12122,6 +11441,15 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv evaluated_chunk_of(&schema, first_metric, 0, vec![Some(1.0), Some(2.0)], true), ) .unwrap(); + let first = execution + .next_chunk() + .unwrap() + .expect("first series output"); + assert_eq!( + decided(std::slice::from_ref(&first)), + vec![(0, Some(100.0)), (1, Some(200.0))] + ); + let second_metric = SeriesLabels::try_new(Some("requests_total".to_string().into()), Vec::new()).unwrap(); execution @@ -12130,8 +11458,6 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv evaluated_chunk_of(&schema, second_metric, 0, vec![Some(3.0), Some(4.0)], true), ) .unwrap(); - // The merge collects both mapped series and reports the collision when its input ends. - execution.end_of_input().unwrap(); let error = execution .next_chunk() .expect_err("the output label set and occupied positions conflict"); @@ -12140,7 +11466,8 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv // scalar broadcast produced no label set of its own, so the rendering is empty and the two // series are told apart only by what they produced. assert!( - message.contains("vector cannot contain metrics with the same labelset"), + message.contains("vector cannot contain metrics with the same labelset") + && message.contains("two series hold a value of field 0"), "expected the collision the node produces, got: {message:?}" ); assert_eq!( @@ -12155,6 +11482,84 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv ); } +/// A literal arithmetic broadcast hands out the label set its vector side already checked, so it +/// keeps no [`Emitted`] table of its own — and the window below it keeps its own. +/// +/// A table is what refuses a second series collapsing onto one label set, and the window is where a +/// dropped name makes that collapse happen. A second copy above it would record every label set the +/// window hands out to decide what the window has already decided. +#[test] +fn a_literal_arithmetic_over_a_name_dropping_window_keeps_no_emitted_table() { + let grid = grid(0, 60_000, 2); + let streaming = || { + window_with( + evaluated_input(Arc::clone(&grid)), + RangeFunction::Increase, + 120_000, + Some(Arc::clone(&grid)), + Arc::clone(&grid), + ) + }; + let collecting = || { + window_with( + map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]), + RangeFunction::Increase, + 120_000, + Some(Arc::clone(&grid)), + Arc::clone(&grid), + ) + }; + // `increase` drops the metric name, so both shapes stand on a window whose table already covers + // the identities this node would hand out -- and the operation is neither `rate` nor a + // multiplication, which is not what decides it. + assert_eq!( + emitted_tables(broadcast_right(streaming(), 2.0, BinaryOp::Add, false)), + (false, true), + "a streaming window repeats its table under a constant add" + ); + assert_eq!( + emitted_tables(broadcast_left(2.0, streaming(), BinaryOp::Add, false)), + (false, true), + "a constant on the left is the same one-to-one map" + ); + assert_eq!( + emitted_tables(broadcast_right(collecting(), 2.0, BinaryOp::Mul, false)), + (false, true), + "a collecting window repeats its table too" + ); + // A window that keeps the metric name hands out identities the operation has not seen: the + // multiplication drops the name, so two names of one tag set become one label set above it. + assert_eq!( + emitted_tables(broadcast_right( + window_with( + evaluated_input(Arc::clone(&grid)), + RangeFunction::LastOverTime, + 120_000, + Some(Arc::clone(&grid)), + Arc::clone(&grid), + ), + 2.0, + BinaryOp::Mul, + false, + )), + (true, true), + "a name-preserving window leaves the collision to the node above it" + ); + // The vector side is the source itself, so nothing below this node dropped a name and this node + // is the only one that can refuse: `a_broadcast_refuses_two_series_that_share_a_label_set_and_ + // releases_the_reservation` is the refusal that rests on the table it keeps. + assert_eq!( + emitted_tables(broadcast_right( + evaluated_input(Arc::clone(&grid)), + 2.0, + BinaryOp::Mul, + false, + )), + (true, false), + "a source under the broadcast leaves it the only node that can refuse" + ); +} + /// What a literal arithmetic broadcast hands on under a name-dropping window keeps its positions: /// series that differ only by name hold different ones, so nothing collides, each hands over its own /// close, and the constant reaches every value that is there. @@ -12322,24 +11727,23 @@ fn a_literal_arithmetic_over_a_collecting_window_hands_on_its_flush() { assert_eq!(pool.reserved(), 0); } -/// The merge over a name-dropping window refuses two series that collapse onto one label set, and -/// it does so before the aggregate above can fold them into one group. +/// The window below a literal arithmetic broadcast refuses two series that collapse onto one label +/// set, and it does so before the aggregate above can fold them into one group. /// -/// `rate` drops the metric name, so the two series below leave the window under one label set: the -/// merge collecting the window's output is where the pair is refused. That placement is what keeps -/// the refusal upstream -- a duplicate the partial aggregate read would land in one group and answer -/// a wrong value instead of refusing, and the reservations would be held until the fold answered. +/// The broadcast keeps no table for this shape, so the refusal is the window's -- which is also what +/// keeps it upstream: a duplicate the partial aggregate read would land in one group and answer a +/// wrong value instead of refusing, and the reservations would be held until the fold answered. #[test] -fn a_merge_over_a_name_dropping_window_refuses_before_the_aggregate() { +fn a_window_under_a_literal_arithmetic_refuses_the_collision_it_collapses() { let out_grid = grid(0, 60_000, 5); let tree = NativeTree::try_new(partial_sum(broadcast_right( - merge(window_with( + window_with( input(), RangeFunction::Rate, 300_000, None, Arc::clone(&out_grid), - )), + ), 2.0, BinaryOp::Mul, false, @@ -12367,8 +11771,8 @@ fn a_merge_over_a_name_dropping_window_refuses_before_the_aggregate() { execution.next_state().unwrap().is_none(), "the partial aggregate answered before its input ended" ); - // The second series is a different one to the scan and the same one to `rate`: the merge over - // the window is where the two meet, and the broadcast above it collects nothing of its own. + // The second series is a different one to the scan and the same one to `rate`: the window is + // where the two meet, and the broadcast above it has no table to notice it. execution .push_raw( InputId::new(0), @@ -12382,9 +11786,6 @@ fn a_merge_over_a_name_dropping_window_refuses_before_the_aggregate() { ), ) .unwrap(); - // The merge holds both series until its input ends, and then reports the pair it cannot file - // under one label set. - execution.end_of_input().unwrap(); let error = match execution.next_state() { Ok(None) => panic!("the second series settled nothing at all"), Ok(Some(_)) => panic!("a second series left the window under the first one's label set"), @@ -12393,11 +11794,15 @@ fn a_merge_over_a_name_dropping_window_refuses_before_the_aggregate() { let message = error.to_string(); assert!( message.contains("vector cannot contain metrics with the same labelset") - && message.contains("host=\"web-1\""), - "expected the merge's refusal, got: {message:?}" + && message.contains("two series hold a value of field 0"), + "expected the window's refusal, got: {message:?}" + ); + // The node the refusal comes from is the one named: the window under the broadcast's left + // operand, not the broadcast that hands its answer on. + assert!( + message.contains("native.window at root.input.left"), + "the refusal does not name the window that made it: {message:?}" ); - // The refusal is upstream of the fold: a duplicate it read would land in one group and answer a - // wrong value instead of refusing. assert_eq!( count(&metrics, "root", NodeMetric::OUTPUT_BATCHES), 0, @@ -12415,16 +11820,17 @@ fn a_merge_over_a_name_dropping_window_refuses_before_the_aggregate() { ); } -/// A window that keeps the metric name leaves the collision to the merge above the broadcast. +/// A window that keeps the metric name leaves the collision to the broadcast, which still keeps the +/// table that refuses it. /// /// `last_over_time` reports the name it read, so the two series stay distinct under it; the /// multiplication drops the name, and the two then hold one label set at the same positions. Nothing -/// below the merge has collected that identity, so the merge above this node is where it is refused. +/// below this node has checked that identity, so this node is where it is refused. #[test] -fn a_name_preserving_window_leaves_the_collision_to_the_merge_above_the_broadcast() { +fn a_name_preserving_window_leaves_the_collision_to_the_broadcast() { let grid = grid(0, 60_000, 2); let schema = evaluated_schema(Arc::clone(&grid)); - let tree = NativeTree::try_new(merge(broadcast_right( + let tree = NativeTree::try_new(broadcast_right( window_with( evaluated_input(Arc::clone(&grid)), RangeFunction::LastOverTime, @@ -12435,7 +11841,7 @@ fn a_name_preserving_window_leaves_the_collision_to_the_merge_above_the_broadcas 2.0, BinaryOp::Mul, false, - ))) + )) .unwrap(); let pool = pool(); let mut execution = executor(&tree, Arc::clone(&pool)); @@ -12451,10 +11857,13 @@ fn a_name_preserving_window_leaves_the_collision_to_the_merge_above_the_broadcas ), ) .unwrap(); - // The merge collects what the broadcast hands over, so nothing is answered yet. - assert!( - execution.next_chunk().unwrap().is_none(), - "the merge answered before its input ended" + let first = execution + .next_chunk() + .unwrap() + .expect("the first series answers under its own name dropped"); + assert_eq!( + decided(std::slice::from_ref(&first)), + vec![(0, Some(2.0)), (1, Some(4.0))] ); execution .push_evaluated( @@ -12468,14 +11877,14 @@ fn a_name_preserving_window_leaves_the_collision_to_the_merge_above_the_broadcas ), ) .unwrap(); - execution.end_of_input().unwrap(); let error = execution .next_chunk() .expect_err("the two names become one label set at the positions both hold"); let message = error.to_string(); assert!( - message.contains("vector cannot contain metrics with the same labelset"), - "expected the collision the merge produces, got: {message:?}" + message.contains("vector cannot contain metrics with the same labelset") + && message.contains("two series hold a value of field 0"), + "expected the collision the broadcast produces, got: {message:?}" ); assert_eq!( pool.reserved(), @@ -12572,6 +11981,7 @@ fn a_histogram_window_under_literal_arithmetic_never_expands_occupancy() { } else { broadcast_right(window(), 2.0, op, false) }; + assert_eq!(emitted_tables(root.clone()), (false, true)); let tree = NativeTree::try_new(root).unwrap(); let pool = pool(); let collector = common_query::promql_annotations::PromqlAnnotationCollector::default(); @@ -13860,8 +13270,8 @@ fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() { for histogram in [false, true] { let grid = grid(1_000, 1_000, 2); let input = evaluated_input(Arc::clone(&grid)); - // Both sides are varying scalars: the broadcast retains one scalar series after producing - // the parameter. + // Both sides are varying scalars: the broadcast retains one scalar series and emitted + // label history after producing the parameter. let parameter = scalar_binary( eval_time(None, Arc::clone(&grid)), eval_time(None, Arc::clone(&grid)), @@ -13957,6 +13367,85 @@ fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() { } } +/// `scalar()` releases its own retained input after producing, but a map below it still holds +/// charged emitted-label history. Consuming the parameter must drop that subtree as well. +#[test] +fn dynamic_parameters_drop_scalar_of_map_history_before_terminal_close() { + for histogram in [false, true] { + let grid = grid(0, 1_000, 2); + let schema = evaluated_schema(Arc::clone(&grid)); + let input = evaluated_input(Arc::clone(&grid)); + let parameter = scalar_of(map( + evaluated_input_with_source_index(Arc::clone(&grid), 1), + vec![MathOp::Abs], + )); + let root = NativeNode { + output: input.output.clone(), + expr: if histogram { + NativeExpr::ParameterizedHistogram { + input: Box::new(input), + op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)), + parameters: vec![parameter], + } + } else { + NativeExpr::ParameterizedTopK { + input: Box::new(input), + op: BoundVectorOp::TopK { + k: 0.0, + top: true, + group: Grouping::By(Arc::from([])), + }, + parameter: Box::new(parameter), + } + }, + }; + let tree = NativeTree::try_new(root).unwrap(); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = metrics_of(&execution); + let parameter_name = if histogram { + "root.parameter0" + } else { + "root.parameter" + }; + let map_name = format!("{parameter_name}.input"); + execution + .push_evaluated( + InputId::new(1), + evaluated_chunk(&schema, 0, vec![1.0, 1.0], true), + ) + .unwrap(); + assert!(execution.next_chunk().unwrap().is_none()); + let map_current = format!("native.{map_name}.current_reserved_bytes"); + assert!( + gauge(&metrics, &map_current) > 0, + "map did not retain emitted history" + ); + // Drive exactly the finishing stage that produces the scalar, before the executor closes. + assert!(execution.live_mut().unwrap().pull(true).unwrap().is_none()); + assert_eq!(gauge(&metrics, &map_current), 0); + assert!( + gauge( + &metrics, + &format!("native.p0.{map_name}.peak_reserved_bytes") + ) > 0 + ); + assert_eq!(count(&metrics, &map_name, NodeMetric::OUTPUT_BATCHES), 1); + let root = &execution.state().live.as_ref().unwrap().root; + match &root.op { + RuntimeOp::ParameterizedTopK { parameter, .. } => assert!(parameter.is_none()), + RuntimeOp::ParameterizedHistogram { parameters, .. } => { + assert!(parameters[0].is_none()) + } + _ => panic!("expected dynamic operator"), + } + assert_eq!(execution.state().reserved_bytes(), pool.reserved()); + execution.live_mut().unwrap().root.finish_windows().unwrap(); + drop(execution); + assert_eq!(pool.reserved(), 0); + } +} + /// A settled parameter with an open window must report the window's finish error before dropping it. #[test] fn dynamic_parameters_finish_windows_before_dropping_the_subtree() { @@ -15327,51 +14816,17 @@ fn count_values_reserves_a_group_per_field_sample() { /// `label_replace` runs in a map: each series is relabelled as it passes, and two series it gives /// one label set collide where both hold a value, as Prometheus' same-labelset check refuses. -/// -/// The map rewrites a label its input carries, so the canonical pass collects what it hands out: the -/// merge above the rewrite is where the two pieces are read against each other. #[test] fn a_label_rewrite_relabels_series_and_refuses_a_shared_point() { let grid = grid(0, 60_000, 2); let schema = evaluated_schema(Arc::clone(&grid)); let rewrite = LabelRewrite::replace("job", "all", "job", ".*").unwrap(); - let tree = |merge_above: bool| { - let rewritten = map_ops( - evaluated_input(Arc::clone(&grid)), - vec![MapOp::Label(Arc::new( - LabelRewrite::replace("job", "all", "job", ".*").unwrap(), - ))], - ); - NativeTree::try_new(if merge_above { - merge(rewritten) - } else { - rewritten - }) - .unwrap() - }; - assert_eq!(rewrite.destination(), "job"); - - // The rewrite itself: one series passes and leaves under the label it wrote. - let mut relabelled = executor(&tree(false), pool()); - relabelled - .push_evaluated( - InputId::new(0), - evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(1.0), Some(2.0)], - true, - ), - ) - .unwrap(); - let first = relabelled - .next_chunk() - .unwrap() - .expect("the relabelled series"); - assert_eq!(first.series.metric, named_metric("a", "all")); - - let mut execution = executor(&tree(true), pool()); + let tree = NativeTree::try_new(map_ops( + evaluated_input(Arc::clone(&grid)), + vec![MapOp::Label(Arc::new(rewrite))], + )) + .unwrap(); + let mut execution = executor(&tree, pool()); execution .push_evaluated( InputId::new(0), @@ -15384,17 +14839,21 @@ fn a_label_rewrite_relabels_series_and_refuses_a_shared_point() { ), ) .unwrap(); + let first = execution + .next_chunk() + .unwrap() + .expect("the relabelled series"); + assert_eq!(first.series.metric, named_metric("a", "all")); execution .push_evaluated( InputId::new(0), evaluated_chunk_of(&schema, named_metric("a", "y"), 1, vec![Some(3.0)], true), ) .unwrap(); - execution.end_of_input().unwrap(); let error = execution.next_chunk().unwrap_err().to_string(); assert!( error.contains("vector cannot contain metrics with the same labelset") - && error.contains("job=\"all\""), + && error.contains("two series hold a value of field 0 at position 1"), "{error}" ); } diff --git a/src/promql/src/native/mod.rs b/src/promql/src/native/mod.rs index 0012c28b03d..3f561281e3d 100644 --- a/src/promql/src/native/mod.rs +++ b/src/promql/src/native/mod.rs @@ -34,6 +34,7 @@ pub mod display; pub mod execute; pub mod metrics; pub mod plan; +pub(crate) mod runs; pub mod segment; pub(crate) mod split; pub(crate) mod state; diff --git a/src/promql/src/native/plan.rs b/src/promql/src/native/plan.rs index 7de7de907fe..f30a7e67d5f 100644 --- a/src/promql/src/native/plan.rs +++ b/src/promql/src/native/plan.rs @@ -1307,10 +1307,9 @@ impl NativeNode { /// A canonical label set is the identity a [`NativeExpr::Merge`] groups by: the metric name plus the /// ordinary labels, which is exactly what [`SeriesLabels`] holds. One scan partition holds one /// series per such set - a stored series is one identity, and the scan splits rows by storage key -/// rather than by label set - so only a computation that *writes* an existing label, or drops one, -/// can turn two of those identities into one. A `label_replace` or a `label_join` writes one, and it -/// may name the very label that told its input's series apart; a range function that drops the -/// metric name, and `timestamp` in either of its forms, drop one. +/// rather than by label set - so only a computation that *writes* an existing label can turn two of +/// those identities into one. A `label_replace` or `label_join` does exactly that, and it may name +/// the very label that told its input's series apart. /// /// The rewrite is judged by the labels it writes and the ones its input carries, which is why the /// caller supplies what it knows about each source's tags, indexed by the `source_index` the leaves @@ -1332,47 +1331,23 @@ impl NativeNode { /// /// Everything else either keeps the set it read or answers a set of its own: /// -/// - A map of math drops the metadata labels, and so do a histogram map, a parameterized map, an -/// arithmetic or `bool` scalar broadcast, a range function that computes its result and a -/// `timestamp` in either of its forms. Such a node is judged by two facts -/// ([`metadata_drop_keeps_series_apart`]): the labels it and its input report, and the scan's own -/// key columns. A source bound to one metric names every series it hands over, so the name the -/// drop removes told none of them apart, and a source that may hand over several is collected; and -/// because the drop removes every empty label value with the name, a key column that holds `NULL` -/// in one series and `''` in another - an absent and an empty value, which is one label set once -/// the empty one is gone - is collected too. `keys_survive` is that second fact, stated by the -/// caller for the scan below the tree -/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop), which is -/// the same rule the physical per-series scan applies); a caller that cannot state it passes -/// `false`, and the conservative answer is the one that collects. +/// - A map of math drops the metric name, and an arithmetic broadcast does too, but within one +/// metric the remaining tags are the primary key: two stored series of one metric that agree on +/// every other label are the same series, so dropping the name cannot make them one. Each of +/// those nodes hands its input's answer on rather than claiming one of its own. /// - An `Aggregate` answers its groups, a `Vector` that folds series answers its result sets, and a /// `Binary` that reads whole vectors files its rows by the label set a sample *ends* under and /// refuses two values of a field at one point. A `Set` does the same, and `info` merges the /// results that share a label set. Each answers one series per set it names. -/// - A `Window` that drops the metadata labels, and the two timestamps, claim one of their own: the -/// name is part of the identity a merge groups by, so two series that shared every other label are -/// one label set from there on - unless the drop tells them apart by the same two-fact rule the -/// kernels above are judged by. -/// - The empty-value half of that proof is answered by a [`Merge`](NativeExpr::Merge) at or below the -/// drop ([`canonical_below`]): a merge hands over one series per canonical label set - the -/// comparison that treats an empty label value as no label - so no drop above it can fold two of -/// its series by removing an empty value. What a merge does not answer is the metadata labels -/// themselves, which the labels half below still has to prove. That is what keeps a chain of drops -/// over one metric to the one collection at its bottom rather than one per drop. -/// - A node that hands its input's series over unchanged - a selection, a name-keeping window, -/// `topk` and its ranking siblings - passes the answer on: an alias below it is still an alias -/// above it. +/// - A node that hands its input's series over unchanged - a selection, a window, a timestamp, a +/// parameterized map, `topk` and its ranking siblings - passes the answer on: an alias below it is +/// still an alias above it. /// /// This is the fact [`merge_label_rewrites`](crate::native::canonicalize::merge_label_rewrites) /// inserts a `Merge` for: a consumer that reads a whole label set cannot answer from pieces, and this /// says where pieces of one set can exist at all. A `Merge` is the collection itself, so it answers /// `false` - which is what makes the pass idempotent. -pub fn duplicates_canonical_labels( - node: &NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> bool { +pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec]) -> bool { match &node.expr { NativeExpr::Input(_) | NativeExpr::Scalar(_) @@ -1380,50 +1355,13 @@ pub fn duplicates_canonical_labels( | NativeExpr::VectorOf { .. } => false, // The collection itself: one series per label set comes out of it. NativeExpr::Merge { .. } => false, - // These hand each series over as they read it, so what aliases below them still aliases -- - // except `timestamp`, which reports a sample's own time under the metadata labels dropped - // and is therefore a source of duplicates in its own right. A merge below it already hands - // over one series per label set, so nothing the drop can fold is left there. - NativeExpr::Instant { input, op } => { - (op.value == InstantValue::Timestamp - && !metadata_drop_keeps_series_apart( - input, - source_tags, - source_metrics, - keys_survive, - )) - || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) - } - // A window that drops the metadata labels can fold two series that agreed on every other - // label into one set, so it is a source of duplicates whether or not its input was one - - // unless the labels and the scan's key columns provably tell those series apart still, or a - // merge below it already collected them. - NativeExpr::Window { input, op } => { - (op.function.drops_metric_name() - && !metadata_drop_keeps_series_apart( - input, - source_tags, - source_metrics, - keys_survive, - )) - || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) - } - // `timestamp` always drops the metadata labels, so two series it reads can become one set - - // unless the labels below it are all bound to one metric and the scan's keys survive the - // drop. - NativeExpr::Timestamp { input } => { - !metadata_drop_keeps_series_apart(input, source_tags, source_metrics, keys_survive) - || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) - } - // A parameterized map is a math kernel - `round`, `clamp` - and drops the metadata labels as - // a math map does, so it answers for the same reason. Its parameter is a scalar and carries - // no labels of the input's. - NativeExpr::ParameterizedMap { input, .. } => { - !metadata_drop_keeps_series_apart(input, source_tags, source_metrics, keys_survive) - || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) - } - NativeExpr::ParameterizedTopK { input, .. } => { - duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + // These hand each series over as they read it, so what aliases below them still aliases. + NativeExpr::Instant { input, .. } + | NativeExpr::Window { input, .. } + | NativeExpr::Timestamp { input } + | NativeExpr::ParameterizedMap { input, .. } + | NativeExpr::ParameterizedTopK { input, .. } => { + duplicates_canonical_labels(input, source_tags) } // `scalar()` answers one value, which names no label set at all. NativeExpr::ScalarOf { .. } => false, @@ -1435,21 +1373,9 @@ pub fn duplicates_canonical_labels( // leaves every series' identity distinct. `map_writes_existing_label` is that question, and // an input whose labels this walk cannot derive is collected rather than trusted. Even a // whole-vector consumer can emit distinct series whose labels a later rewrite makes equal. - // A math or histogram kernel in the same map drops the metadata labels, which is a source of - // duplicates of its own unless the labels and the scan's keys tell its input's series apart - - // see the doc comment above. NativeExpr::Map { input, op } => { - (op.ops - .iter() - .any(|operation| matches!(operation, MapOp::Math(_) | MapOp::Histogram(_))) - && !metadata_drop_keeps_series_apart( - input, - source_tags, - source_metrics, - keys_survive, - )) - || map_writes_existing_label(input, op, source_tags).unwrap_or(true) - || duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) + map_writes_existing_label(input, op, source_tags).unwrap_or(true) + || duplicates_canonical_labels(input, source_tags) } // A fold answers one series per group, which is one label set per group. NativeExpr::Aggregate { .. } => false, @@ -1458,49 +1384,23 @@ pub fn duplicates_canonical_labels( BoundVectorOp::TopK { .. } | BoundVectorOp::LimitK { .. } | BoundVectorOp::LimitRatio { .. } - | BoundVectorOp::Sort(_) => { - duplicates_canonical_labels(input, source_tags, source_metrics, keys_survive) - } + | BoundVectorOp::Sort(_) => duplicates_canonical_labels(input, source_tags), // `absent` reports one series, `count_values` one per distinct value, and the histogram // functions fold the bucket series of one histogram into one series. BoundVectorOp::Absent { .. } | BoundVectorOp::Histogram(_) | BoundVectorOp::CountValues { .. } => false, }, - NativeExpr::Binary { - left, - right, - matching, - } => { + NativeExpr::Binary { left, right, .. } => { let vector = |side: &NativeNode| side.output.samples().is_some(); - match (vector(left), vector(right)) { + if vector(left) && vector(right) { // Two vectors are matched by label, and the kernel files its rows by the label set a // sample ends under, refusing two values of one field at one point. - (true, true) => false, - // A broadcast reaches each series of its vector side as it is. An arithmetic one and - // one that reports a `bool` drop the metadata labels, which is a source of - // duplicates of its own unless the labels and the scan's keys tell its series apart; - // a filtering comparison keeps the set it read. A merge below the vector side already - // handed over one series per label set, so the drop has nothing left to fold there. - (true, false) | (false, true) => { - let carried = if vector(left) { left } else { right }; - let drops_metadata = !matching.op.is_comparison() || matching.return_bool; - (drops_metadata - && !metadata_drop_keeps_series_apart( - carried, - source_tags, - source_metrics, - keys_survive, - )) - || duplicates_canonical_labels( - carried, - source_tags, - source_metrics, - keys_survive, - ) - } - // Two scalars name no label set at all. - (false, false) => false, + false + } else { + // A broadcast reaches each series of its vector side as it is. + let carried = if vector(left) { left } else { right }; + duplicates_canonical_labels(carried, source_tags) } } // A set operation keeps each side's samples and files them by result label set, refusing two @@ -1547,313 +1447,6 @@ fn map_writes_existing_label( } Some(existing) } - -/// Whether the metadata labels a rewrite drops are the only ones telling `node`'s series apart. -/// -/// A range function that computes its result and `timestamp` in either of its forms drop -/// [`PROMQL_METADATA_LABELS`] - the metric name, `__type__` and `__unit__` - and a consumer above -/// them that reads a whole label set has to establish those series cannot collide once the three are -/// gone. Two series of one input collide only if they differed in one of the three alone, -/// so the drop is injective - and nothing has to be collected - when neither can tell two of them -/// apart: -/// -/// - The name: a source bound to one metric names every series it hands over, so the name is the -/// same for all of them and removing it merges nothing (see [`single_metric_name`]). A source that -/// may hand over several - one selector that discovered several metrics, or a caller that cannot -/// tell - is not proved, and the conservative answer is the one that collects. -/// - `__type__` and `__unit__`: both are removed along with the name, and neither is a label this -/// walk can prove constant across a source's series. They are ordinary labels when a source -/// carries them as tags or a rewrite writes them, and two series that agree on everything else are -/// one label set once they are gone. Their presence anywhere in `node`'s output is therefore -/// enough to withhold the proof. -/// -/// A node whose labels already lost the name - `abs`, a name-dropping window below it - has no name -/// of its own left to remove, and what its surviving labels tell apart is then only as good as the -/// metrics below it: a drop that happened on the way there removed a name without asking this walk, -/// so the proof asks instead that every source it reads is bound to the same one metric -/// ([`single_source_metric`]). Over a selector that discovered several metrics that drop folds them -/// together, and the answer is the one that collects. -/// -/// `false` when the walk cannot derive `node`'s labels at all, because an unknown set proves nothing. -/// -/// This is the whole of the drop's proof: every arm of [`duplicates_canonical_labels`] that decides -/// whether a collection is needed asks it, and so does [`unifies_its_own_contributions`] at run -/// time. What it cannot see is a key column that holds `NULL` in one series and `''` in another, -/// both of which decode to no label once the empty value is dropped with the name. That fact is not -/// derivable from a tree - it is read from the resolved table - so the caller states it as -/// `keys_survive`, using the same rule the physical per-series scan applies -/// ([`keys_survive_metadata_drop`](crate::extension_plan::keys_survive_metadata_drop)). -/// [`leaves_a_drop_to_the_storage_keys`] is what makes the statements collected for that reason -/// countable. -fn metadata_drop_keeps_series_apart( - node: &NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> bool { - // The empty-value half is asked first, because it is the cheap one and because a merge at or - // below `node` answers it for the tree ([`canonical_below`]): a merge hands over one series per - // canonical label set - the comparison that treats an empty label value as no label - so two of - // its series are already distinct under it, and no drop above can fold them by removing an empty - // value. What a merge does *not* answer is the metadata labels themselves: it keeps two series - // that differ in the name, `__type__` or `__unit__`, and those are exactly what the drop - // removes, so the labels half still has to hold on its own. - // - // A caller that cannot state the key half - one holding no table, or a tree whose scans - // disagree - passes `keys_survive = false`, and only a merge below it can then carry the drop. - (keys_survive || canonical_below(node)) - && metadata_labels_keep_series_apart(node, source_tags, source_metrics) -} - -/// Whether the labels alone tell two of `node`'s series apart once the metadata labels are dropped. -/// -/// This is the half of [`metadata_drop_keeps_series_apart`] a tree can answer about itself; the other -/// half is the empty values the drop removes - the scan's key columns, or a merge that has already -/// told those apart. See that function's doc comment for the rule itself. -fn metadata_labels_keep_series_apart( - node: &NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], -) -> bool { - let Some(tags) = fused_output_tags(node, source_tags) else { - return false; - }; - if tags - .iter() - .any(|tag| tag == "__type__" || tag == "__unit__") - { - return false; - } - if !tags.iter().any(|tag| tag == METRIC_NAME) { - // No name is left to lose, so what the surviving labels tell apart is decided by the sources - // below: every one of them reading the same single metric is what says a name dropped on the - // way here could not have folded two metrics together either. `abs` and a name-dropping - // window both drop the name without asking, so without that fact nothing below is proved. - return single_source_metric(node, source_metrics).is_some(); - } - single_metric_name(node, source_metrics).is_some() -} - -/// Whether a [`NativeExpr::Merge`] collects the series `node` hands over, at it or below it. -/// -/// A merge holds one series per canonical label set, so the series above one are not stored series -/// any more: a merge with nothing to merge hands its input's pieces through, and one that does -/// merge keeps the first piece's labels and drops the pieces it folded, so what comes out is unique -/// under the label set Prometheus compares (`label_set` in `grid_series`) - the one that treats an -/// empty label value as no label. Two pieces that came out of one merge are therefore distinct under -/// that comparison even though their rows may still spell an empty value, which is what makes a -/// second drop above the merge unable to fold them by removing an empty value. What it does not make -/// impossible is the rest of such a drop: a merge keeps two series that differ in `__name__`, -/// `__type__` or `__unit__`, and removing those is exactly what a metadata drop does. -/// -/// Only the children that carry samples are searched: a parameter is a scalar or a vector of -/// constants, and a merge below one reads series this node's own samples did not come from. -fn canonical_below(node: &NativeNode) -> bool { - matches!(node.expr, NativeExpr::Merge { .. }) - || node - .expr - .children() - .into_iter() - .filter(|child| matches!(child.output, NativeType::Samples(_))) - .any(canonical_below) -} - -/// Whether `node` drops [`PROMQL_METADATA_LABELS`] on its own over series whose labels are proved to -/// tell apart and whose keys do not. -/// -/// This is the question of what the drop costs: the labels half of the proof -/// ([`metadata_labels_keep_series_apart`]) holds, nothing at or below the drop has collected already -/// ([`canonical_below`]), and the only reason a collection is still needed is the scan's key columns, -/// a primary-key column that holds `NULL` in one series and `''` in another. Those two decode to an -/// absent and to an empty label, and the drop normalizes them into one. Such a statement is now -/// collected for that reason alone ([`metadata_drop_keeps_series_apart`]), and this is the count -/// that says so. -/// -/// Every node that drops the metadata labels on its own counts: a map of math, a histogram map, a -/// parameterized map and an arithmetic or `bool` scalar broadcast, and the three that rewrite a -/// sample's own value - a name-dropping range function, `timestamp(x)` and a `timestamp` over an -/// evaluated input. -fn drops_metadata_labels_over_one_metric( - node: &NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> bool { - // The statement is counted exactly when the drop is collected for the key columns alone: the - // labels prove it ([`metadata_labels_keep_series_apart`]), nothing at or below it has collected - // already ([`canonical_below`]), and the keys do not survive it - which is the one fact the pass - // read from the resolved table and the reason the merge is there. - let decided_by_the_keys = |carried: &NativeNode| { - !keys_survive - && metadata_labels_keep_series_apart(carried, source_tags, source_metrics) - && !canonical_below(carried) - }; - let dropped = match &node.expr { - NativeExpr::Instant { input, op } => { - op.value == InstantValue::Timestamp && decided_by_the_keys(input) - } - NativeExpr::Timestamp { input } => decided_by_the_keys(input), - NativeExpr::Window { input, op } => { - op.function.drops_metric_name() && decided_by_the_keys(input) - } - NativeExpr::ParameterizedMap { input, .. } => decided_by_the_keys(input), - NativeExpr::Map { input, op } => { - op.ops - .iter() - .any(|operation| matches!(operation, MapOp::Math(_) | MapOp::Histogram(_))) - && decided_by_the_keys(input) - } - // A broadcast reaches each series of its vector side as it is; an arithmetic one and one - // that reports a `bool` drop the metadata labels, a filtering comparison keeps them. - NativeExpr::Binary { - left, - right, - matching, - } => { - let carried = match ( - left.output.samples().is_some(), - right.output.samples().is_some(), - ) { - (true, false) => Some(left.as_ref()), - (false, true) => Some(right.as_ref()), - _ => None, - }; - carried.is_some_and(|carried| { - (!matching.op.is_comparison() || matching.return_bool) - && decided_by_the_keys(carried) - }) - } - _ => false, - }; - dropped - || node.expr.children().into_iter().any(|child| { - drops_metadata_labels_over_one_metric(child, source_tags, source_metrics, keys_survive) - }) -} - -/// Whether `tree` holds a node that drops the metadata labels over series whose labels are proved to -/// tell apart and whose keys do not (see [`drops_metadata_labels_over_one_metric`]). -/// -/// A statement this answers `true` for is one the canonical pass collects for the key columns alone: -/// the labels prove the drop injective, nothing below it has collected already, and the scan below it -/// keys its series by a column that can hold `NULL` in one series and `''` in another -/// (`keys_survive` is `false`, and the caller states that fact from the resolved table). Those -/// statements are the price of reading that fact instead of the label-set check the physical scan -/// only states for itself, so the caller counts them by this answer. -pub fn leaves_a_drop_to_the_storage_keys( - tree: &NativeTree, - source_tags: &[Vec], - source_metrics: &[Option>], - keys_survive: bool, -) -> bool { - drops_metadata_labels_over_one_metric(tree.root(), source_tags, source_metrics, keys_survive) -} - -/// The one metric every source `node` reads is bound to, or `None` when they are not all one. -/// -/// This is the facts-free question about the leaves alone: a node can drop the metric name on the -/// way (a math map, a range function) whatever its sources hold, so what the surviving labels tell -/// apart is only as good as the set of metrics below them. -fn single_source_metric( - node: &NativeNode, - source_metrics: &[Option>], -) -> Option> { - let mut pending = vec![node]; - let mut metric: Option<&Arc> = None; - while let Some(node) = pending.pop() { - match &node.expr { - // A source that may hand over several metrics proves nothing about their names. - NativeExpr::Input(input) => { - let bound = source_metrics.get(input.source_index)?.as_ref()?; - match metric { - None => metric = Some(bound), - Some(seen) if seen == bound => {} - Some(_) => return None, - } - } - NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => {} - NativeExpr::Instant { input, .. } - | NativeExpr::Window { input, .. } - | NativeExpr::Map { input, .. } - | NativeExpr::Timestamp { input } - | NativeExpr::VectorOf { input } - | NativeExpr::Merge { input } - | NativeExpr::Aggregate { input, .. } - | NativeExpr::Vector { input, .. } - | NativeExpr::ScalarOf { input } => pending.push(input), - NativeExpr::ParameterizedMap { - input, parameters, .. - } - | NativeExpr::ParameterizedHistogram { - input, parameters, .. - } => { - pending.push(input); - pending.extend(parameters.iter()); - } - NativeExpr::ParameterizedTopK { - input, parameter, .. - } => { - pending.push(input); - pending.push(parameter); - } - NativeExpr::Binary { left, right, .. } - | NativeExpr::Set { left, right, .. } - | NativeExpr::Info { - base: left, - info: right, - .. - } => { - pending.push(left); - pending.push(right); - } - } - } - metric.cloned() -} - -/// The one metric every series `node` hands over is named, or `None` when this walk cannot say. -/// -/// Only the nodes that hand their input's label set on unchanged can carry a name: every kernel that -/// reports a value of its own drops it, so a name above one of those is one a rewrite wrote. What is -/// left is the boundary the leaves are bound to - where `source_metrics` answers, indexed by the -/// `source_index` the leaf reads - and the nodes that pass a set through. -fn single_metric_name(node: &NativeNode, source_metrics: &[Option>]) -> Option> { - match &node.expr { - NativeExpr::Input(input) => source_metrics.get(input.source_index).cloned().flatten(), - // These report the label set they read rather than one of their own, so the name is their - // input's. A map that computes a value or writes a name is not one of them. - NativeExpr::Merge { input } - | NativeExpr::ParameterizedTopK { input, .. } - | NativeExpr::Vector { - input, - op: - BoundVectorOp::TopK { .. } - | BoundVectorOp::LimitK { .. } - | BoundVectorOp::LimitRatio { .. } - | BoundVectorOp::Sort(_), - } => single_metric_name(input, source_metrics), - NativeExpr::Instant { input, op } if op.value != InstantValue::Timestamp => { - single_metric_name(input, source_metrics) - } - NativeExpr::Window { input, op } if !op.function.drops_metric_name() => { - single_metric_name(input, source_metrics) - } - NativeExpr::Map { input, op } - if op.ops.iter().all(|operation| { - matches!( - operation, - MapOp::Label(rewrite) if rewrite.destination() != METRIC_NAME - ) - }) => - { - single_metric_name(input, source_metrics) - } - // A kernel that reports a value of its own drops the name, and a fold, a set operation, an - // `info` lookup and a rewrite writing the name answer a set this walk does not follow. - _ => None, - } -} - /// Whether `node` reads the whole input vector at a point rather than one series at a time. /// /// An `Aggregate` folds every series of its group, a `Binary` and a `Set` pair two vectors' series @@ -1871,16 +1464,11 @@ fn single_metric_name(node: &NativeNode, source_metrics: &[Option>]) -> /// Both a region-side placement and the native route's admission ask this, and they must agree: a /// tree admitted here but refused there becomes a node the physical layer cannot build, and one /// refused here but admitted there would be pushed to a region that holds part of the metric. -/// -/// The walk asks the facts-free form of [`unifies_its_own_contributions`]: a placement reads a tree -/// without the binding that says which metric each of its inputs reads, and every reader of this -/// answer - the admission below, the region-side classification and the locality of a built node - -/// asks it of the same tree, so the conservative answer is the one that keeps them agreeing. pub fn reads_across_series(node: &NativeNode) -> bool { match &node.expr { NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => false, NativeExpr::Window { input, .. } => { - matches!(unifies_its_own_contributions(input, &[], &[]), Some(false)) + matches!(unifies_its_own_contributions(input), Some(false)) || reads_across_series(input) } NativeExpr::Instant { input, .. } @@ -2904,37 +2492,8 @@ fn broadcast_drops_labels(matching: &BinaryMatching) -> bool { /// /// The collection condition is `matches!(answer, Some(false))`: the Window `MergeInput` collector /// at runtime and the `limitk` collector both ask this predicate of their input. -/// -/// `source_tags` and `source_metrics` are the same facts -/// [`duplicates_canonical_labels`] judges a whole label set by, indexed the same way: a node that -/// drops the metadata labels reports fragments only when -/// [`metadata_labels_keep_series_apart`] cannot prove the labels tell its series apart. That is the -/// labels half of the drop's proof, and it is the half a tree can answer about itself. The other half -/// is whether the scan's key columns can spell one series as `NULL` and another as `''`, both of -/// which the drop normalizes into one label set: it is stated for the tree as a whole and is asked by -/// the canonical pass ([`duplicates_canonical_labels`], which reads it as -/// [`metadata_drop_keeps_series_apart`]). Where that half does not hold, the pass has put its merge -/// above this node already, so this predicate is asked of the merge instead, which is why asking the -/// labels half here cannot disagree with the pass about a drop. The statements that half collects -/// are counted by [`leaves_a_drop_to_the_storage_keys`]. -/// -/// This predicate is only ever asked of a node by the consumer directly above it - the `Window` -/// `MergeInput` collector and the `limitk` one, both in [`execute`](crate::native::execute) - and -/// that is exactly where the canonical pass puts its [`NativeExpr::Merge`] when a collection is -/// needed: it collects the input of a `Window`, an `Aggregate`, a `Vector`, a `Map`, a -/// `ParameterizedMap` and each side of a `Binary`/`Set`/`Info` -/// ([`merge_label_rewrites`](crate::native::canonicalize::merge_label_rewrites)). The collector -/// then reads the merge's `Some(true)` - a merge *is* the collection - so the weaker answer this -/// function gives about the node beneath it is never the one acted on. Where the pass left the tree -/// alone, the proof held, and the answer asked here is the same one. A caller without these facts -/// (placement, which reads a tree without a binding) passes empty slices and gets the conservative -/// answer. #[allow(clippy::match_same_arms)] // Keep each expression's contribution guarantee explicit. -pub fn unifies_its_own_contributions( - node: &NativeNode, - source_tags: &[Vec], - source_metrics: &[Option>], -) -> Option { +pub fn unifies_its_own_contributions(node: &NativeNode) -> Option { match &node.expr { // A boundary hands over the pieces some upstream execution produced: what it holds is one // storage series at a time for a scan and one partition's fragments of a label set for an @@ -2951,20 +2510,7 @@ pub fn unifies_its_own_contributions( // arrive one whole storage series at a time, so it answers one series per series it read, // under the label set it read. `timestamp(x)` is the exception: it reports the sample's own // time under the metric name dropped, which can give two series one label set. - NativeExpr::Instant { input, op } => { - if op.value == InstantValue::Timestamp { - // `timestamp` drops the metadata labels, so what it emits aliases unless the drop - // provably tells its series apart. The labels half is asked here; the keys half is - // what the pass collects with, above this node - see this function's docs. - Some(metadata_labels_keep_series_apart( - input, - source_tags, - source_metrics, - )) - } else { - Some(true) - } - } + NativeExpr::Instant { op, .. } => Some(op.value != InstantValue::Timestamp), // The merge is the collection: whatever its input handed over, it groups the fragments by // canonical label set and emits one series per set, so what it emits shares none. NativeExpr::Merge { .. } => Some(true), @@ -2982,46 +2528,20 @@ pub fn unifies_its_own_contributions( // folds each contribution on its own, so what it emits does alias its input's fragments, // and `Some(false)` is exact. NativeExpr::Window { input, op } => { - let child = unifies_its_own_contributions(input, source_tags, source_metrics); - if child.is_none() { - // A boundary hands the pieces of one series over one at a time, and the window folds - // them without ever seeing the whole series: what it emits aliases whatever the - // labels say. - Some(false) - } else if op.function.drops_metric_name() { - // The function drops the metadata labels, so the window answers `Some(true)` exactly - // when the labels say the drop tells its input's series apart. The keys half is the - // pass's to ask: where it fails, the pass has already put its merge above this - // window and the collector reads that merge's answer - see this function's docs. - Some(metadata_labels_keep_series_apart( - input, - source_tags, - source_metrics, - )) - } else { - Some(true) - } + Some(!op.function.drops_metric_name() && unifies_its_own_contributions(input).is_some()) } - // A map can rewrite labels, so two series can end under one label set: what it hands out is - // not one series per set until a canonical merge has collected the set. + // A map can rewrite labels, so two series can end under one label set and the runtime + // records what it wrote in an `Emitted` table. NativeExpr::Map { .. } | NativeExpr::ParameterizedMap { .. } => Some(false), // `topk` hands back the series it read with their labels, so the input's answer is its own - // exactly as for the bound operation. - NativeExpr::ParameterizedTopK { input, .. } => { - unifies_its_own_contributions(input, source_tags, source_metrics) - } + NativeExpr::ParameterizedTopK { input, .. } => unifies_its_own_contributions(input), // A histogram function groups its rows by label set itself, so it emits one series per set. NativeExpr::ParameterizedHistogram { .. } => Some(true), // `scalar()` answers one value per point, which is not several series. NativeExpr::ScalarOf { .. } => Some(true), - // A timestamp reports a position under the metadata labels dropped, and the labels half - // asked here is answered for the same reason the `timestamp` selection's is: the keys half - // is the one the pass turns into a merge above this node - see this function's docs. - NativeExpr::Timestamp { input } => Some(metadata_labels_keep_series_apart( - input, - source_tags, - source_metrics, - )), + // A timestamp reports a position under the metric name dropped. + NativeExpr::Timestamp { .. } => Some(false), NativeExpr::Binary { left, right, @@ -3055,7 +2575,7 @@ pub fn unifies_its_own_contributions( !matches!(node.expr, NativeExpr::Scalar(BoundScalar { .. })) }; let vector = if carries_series(left) { left } else { right }; - unifies_its_own_contributions(vector, source_tags, source_metrics) + unifies_its_own_contributions(vector) } } // A set operation keeps each side's samples and files them by label set, one series per @@ -3083,7 +2603,7 @@ pub fn unifies_its_own_contributions( if whole { Some(true) } else { - unifies_its_own_contributions(input, source_tags, source_metrics) + unifies_its_own_contributions(input) } } } diff --git a/src/promql/src/native/plan/tests.rs b/src/promql/src/native/plan/tests.rs index 5a321907c3f..3ccea9949d7 100644 --- a/src/promql/src/native/plan/tests.rs +++ b/src/promql/src/native/plan/tests.rs @@ -24,8 +24,7 @@ use super::{ AggregateKind, AggregateStage, BoundAggregate, BoundInput, BoundInstant, BoundMap, BoundScalar, BoundVectorOp, BoundWindow, Grouping, InstantValue, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, ParameterizedOp, aggregate_state_binding, duplicates_canonical_labels, - fused_output_tags, leaves_a_drop_to_the_storage_keys, reads_across_series, - unifies_its_own_contributions, + fused_output_tags, reads_across_series, unifies_its_own_contributions, }; use crate::extension_plan::FieldKind; use crate::extension_plan::{FusedSeries, FusedSeriesBinding}; @@ -188,44 +187,6 @@ fn window_over(inner: NativeNode, input_grid: Option>) -> NativeNo window_on(inner, input_grid, grid(0, 60_000, 4)) } -/// An instant selection over `inner` reporting the sample it read, which keeps its identity. -fn instant_sample(inner: NativeNode, out: Arc) -> NativeNode { - NativeNode { - expr: NativeExpr::Instant { - input: Box::new(inner), - op: BoundInstant { - lookback_ms: 300_000, - offset_ms: 0, - at_ms: None, - value: InstantValue::Sample, - output_grid: Arc::clone(&out), - }, - }, - output: evaluated(fields(1), out), - } -} - -/// A window over `inner` whose function keeps the metric name, so it hands its input's identity on. -fn keeping_window_fields(inner: NativeNode, out: Arc, count: usize) -> NativeNode { - NativeNode { - expr: NativeExpr::Window { - input: Box::new(inner), - op: BoundWindow { - function: RangeFunction::LastOverTime, - window: RangeWindow { - range_ms: 300_000, - offset_ms: 0, - anchor_ms: None, - modifier: None, - }, - input_grid: None, - output_grid: Arc::clone(&out), - }, - }, - output: evaluated(fields(count), out), - } -} - /// A window over `inner` that produces `count` fields on `out`, so a consumer of it can be given a /// layout the window itself holds legally. fn window_fields(inner: NativeNode, out: Arc, count: usize) -> NativeNode { @@ -1214,12 +1175,12 @@ fn a_broadcast_side_contributes_no_labels() { // broadcast reaches each series of its vector side as it is, so whether two series can end // under one label set is that side's answer, not the constant's. assert!( - duplicates_canonical_labels(&aliasing(), &source_tags, &[], true), + duplicates_canonical_labels(&aliasing(), &source_tags), "the fixture aliases: a rewrite of a label its input carries can fold two series into one" ); assert_eq!( - duplicates_canonical_labels(&aliasing(), &source_tags, &[], true), - duplicates_canonical_labels(&clamp_max(aliasing()), &source_tags, &[], true), + duplicates_canonical_labels(&aliasing(), &source_tags), + duplicates_canonical_labels(&clamp_max(aliasing()), &source_tags), "a scalar parameter carries no labels, so the map answers its input's identity" ); for node in [ @@ -1229,8 +1190,8 @@ fn a_broadcast_side_contributes_no_labels() { binary(eval_time(), clamp_max(aliasing())), ] { assert_eq!( - duplicates_canonical_labels(&aliasing(), &source_tags, &[], true), - duplicates_canonical_labels(&node, &source_tags, &[], true), + duplicates_canonical_labels(&aliasing(), &source_tags), + duplicates_canonical_labels(&node, &source_tags), "a broadcast keeps the series identity of its vector side" ); } @@ -1243,7 +1204,7 @@ fn a_broadcast_side_contributes_no_labels() { binary(vector_of_one(), aliasing()), ] { assert!( - !duplicates_canonical_labels(&node, &source_tags, &[], true), + !duplicates_canonical_labels(&node, &source_tags), "a vector-vector match answers one series per label set, whatever aliases below it" ); } @@ -2200,25 +2161,15 @@ fn scan_tags() -> Vec> { vec![vec!["host".to_string()]] } -/// The labels a scan bound to one metric reports: its tag, and the name its binding gives every -/// series it hands over, which is not one of its tag columns. -fn named_scan_tags() -> Vec> { - vec![vec!["host".to_string(), "__name__".to_string()]] -} - /// A rewrite that writes a label the input does not carry cannot make two series equal, which is /// what lets a query like `label_replace(x, "dst", "$1", "host", "(.*)")` keep every partition. #[test] fn a_fresh_destination_does_not_alias() { let out = grid(0, 60_000, 4); - // A window that keeps the metric name hands its input's identity on, so the rewrite is the only - // node that could make two series equal. - let inner = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); + let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1); assert!(!duplicates_canonical_labels( &rewritten(inner, "dst", "host"), - &scan_tags(), - &[], - true + &scan_tags() )); } @@ -2230,9 +2181,7 @@ fn a_rewrite_of_an_existing_label_aliases() { let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1); assert!(duplicates_canonical_labels( &rewritten(inner, "host", "host"), - &scan_tags(), - &[], - true + &scan_tags() )); } @@ -2246,26 +2195,20 @@ fn a_rewrite_of_the_metric_name_aliases_a_named_source() { let named = vec![vec!["host".to_string(), "__name__".to_string()]]; assert!(duplicates_canonical_labels( &rewritten(inner, "__name__", "__name__"), - &named, - &[], - true + &named )); // The delete-only form can fold the same two identities, so it is collected too. let inner = input_on(Arc::clone(&out)); assert!(duplicates_canonical_labels( &replaced(inner, "__name__", "", "__name__"), - &named, - &[], - true + &named )); } /// A name write is collected even above a kernel that dropped the name: the scan below `abs`, a /// `timestamp` or a range function can still hold several metrics whose ordinary labels are equal, -/// so writing one name puts those identities under one label set. A fresh ordinary destination over -/// a kernel that keeps the name is still uncollected, because no scan can carry an ordinary label it -/// does not declare -- while over a name-dropping kernel the answer is `true` for the kernel's own -/// sake, which is what [`dropping_the_metric_name_is_a_new_source`] pins. +/// so writing one name puts those identities under one label set. A fresh ordinary destination is +/// still uncollected, because no scan can carry an ordinary label it does not declare. #[test] fn a_name_rewrite_above_a_dropped_name_is_collected() { let out = grid(0, 60_000, 4); @@ -2288,21 +2231,16 @@ fn a_name_rewrite_above_a_dropped_name_is_collected() { ] { assert!( duplicates_canonical_labels( - &replaced(dropped, "__name__", "same", "missing"), - &named, - &[], - true + &replaced(dropped.clone(), "__name__", "same", "missing"), + &named ), "{name}: a name write may alias metrics the kernel left indistinguishable" ); + assert!( + !duplicates_canonical_labels(&rewritten(dropped, "dst", "host"), &named), + "{name}: a fresh ordinary destination cannot alias" + ); } - // The fresh destination is judged against a kernel that keeps the name, which is the only shape - // where the rewrite alone decides the answer. - let keeping = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); - assert!( - !duplicates_canonical_labels(&rewritten(keeping, "dst", "host"), &named, &[], true), - "a fresh ordinary destination cannot alias" - ); } /// The label walk keeps a name a rewrite writes and drops it wherever the kernel does, so the @@ -2474,189 +2412,20 @@ fn an_unknown_source_may_alias() { let inner = window_fields(input_with_fields(1), Arc::clone(&out), 1); assert!(duplicates_canonical_labels( &rewritten(inner, "dst", "host"), - &[], - &[], - true + &[] )); } -/// Dropping the metric name is a source of duplicates in its own right, and a window that keeps it -/// is not. -/// -/// The name is part of the identity a merge groups by, so a kernel that computes a value of its own -/// and reports the name dropped can leave two series under one label set - but only when something -/// below it could be told apart by that name. -/// -/// A scan bound to one metric names every series it hands over, so no name of its tells two of them -/// apart, and the labels are the whole proof: the one metric is enough - as long as the scan's key -/// columns always decode a value. A drop removes every empty label value with the name, so a key -/// column that holds `NULL` in one series and `''` in another leaves both under one label set and -/// that half withholds the proof too. A scan that may hand over several metrics is collected, and so -/// is one carrying `__type__` or `__unit__` among its tags, because only those labels told the two -/// series apart. +/// Dropping the metric name is not aliasing within one metric: the remaining tags are the primary +/// key, so two series that agree on all of them are the same stored series. #[test] -fn dropping_the_metric_name_is_a_new_source() { +fn dropping_the_metric_name_keeps_the_inputs_answer() { let out = grid(0, 60_000, 4); - let single = [Some(Arc::::from("some_metric"))]; - let dropped = window_fields(input_with_fields(1), Arc::clone(&out), 1); - // A caller that says nothing about the metric is not proved. - assert!(duplicates_canonical_labels( - &dropped, - &named_scan_tags(), - &[], - true - )); - // The scan reads one metric, so the name it drops told none of its series apart. + let plain = window_fields(input_with_fields(1), Arc::clone(&out), 1); + assert!(!duplicates_canonical_labels(&plain, &scan_tags())); assert!(!duplicates_canonical_labels( - &dropped, - &named_scan_tags(), - &single, - true - )); - // A source that may hand over several metrics is not proved, and neither is one that carries a - // metadata label of its own: both can hold two series that only that label told apart. - assert!(duplicates_canonical_labels( - &dropped, - &named_scan_tags(), - &[None], - true - )); - for metadata in ["__type__", "__unit__"] { - let mut tags = named_scan_tags(); - tags[0].push(metadata.to_string()); - assert!( - duplicates_canonical_labels(&dropped, &tags, &single, true), - "{metadata}" - ); - } - // The same scan, with a key column that can hold `NULL` in one series and `''` in another: the - // empty value leaves with the name, so the one metric no longer proves the drop - and neither - // does it for `abs` or a `timestamp` above it, which report the name dropped as well. - assert!(duplicates_canonical_labels( - &dropped, - &named_scan_tags(), - &single, - false - )); - assert!(duplicates_canonical_labels( - &map_over(input_on(Arc::clone(&out))), - &named_scan_tags(), - &single, - false - )); - // A window that keeps the name passes its input's answer on instead. - let keeping = keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); - assert!(!duplicates_canonical_labels( - &keeping, - &named_scan_tags(), - &[], - true - )); - // `timestamp` reports a sample's own time under the name dropped, in both of its forms. - let stamps = |inner: NativeNode| NativeNode { - expr: NativeExpr::Timestamp { - input: Box::new(inner), - }, - output: evaluated(fields(1), Arc::clone(&out)), - }; - assert!(duplicates_canonical_labels( - &stamps(input_on(Arc::clone(&out))), - &named_scan_tags(), - &[], - true - )); - assert!(duplicates_canonical_labels( - &stamps(input_on(Arc::clone(&out))), - &named_scan_tags(), - &[None], - true - )); - assert!(!duplicates_canonical_labels( - &stamps(input_on(Arc::clone(&out))), - &named_scan_tags(), - &single, - true - )); - // A sample-preserving selection keeps the name it read, so it is no source of its own. - assert!(!duplicates_canonical_labels( - &instant_sample(input_on(Arc::clone(&out)), Arc::clone(&out)), - &named_scan_tags(), - &[], - true - )); -} - -/// The plan-time counter's question: which statements drop the metadata labels over series whose -/// labels prove them apart and whose keys do not. -/// -/// The labels half is what the walk reads; the key half is stated by the caller, and a drop is -/// counted exactly when the labels prove it, nothing below it has collected already, and the scan's -/// key columns cannot tell the pair apart - a key column holding `NULL` in one series and `''` in -/// another, which is a fact about the physical table. This is the tree half of that count: the -/// planner pairs it with the fact it read from the resolved table. -#[test] -fn a_drop_the_labels_prove_is_counted_only_when_its_keys_do_not() { - let out = grid(0, 60_000, 4); - let single = [Some(Arc::::from("some_metric"))]; - let named = named_scan_tags(); - // `rate(x[5m])` and `abs(abs(x))` over one metric: the drop removes a name every series shares, - // and nothing below either node is collected, so the key columns are what decides the answer. - for keys in [false, true] { - let counted = !keys; - assert_eq!( - counted, - leaves_a_drop_to_the_storage_keys( - &NativeTree::try_new(window_fields(input_with_fields(1), Arc::clone(&out), 1)) - .unwrap(), - &named, - &single, - keys, - ), - "`rate(x[5m])` with keys_survive = {keys}" - ); - assert_eq!( - counted, - leaves_a_drop_to_the_storage_keys( - &NativeTree::try_new(map_over(map_over(input_on(Arc::clone(&out))))).unwrap(), - &named, - &single, - keys, - ), - "`abs(abs(x))` with keys_survive = {keys}" - ); - } - // A source that may hand over several metrics is collected by the labels, so nothing is left to - // the key columns. - assert!(!leaves_a_drop_to_the_storage_keys( - &NativeTree::try_new(window_fields(input_with_fields(1), Arc::clone(&out), 1)).unwrap(), - &named, - &[None], - false, - )); - // A window that keeps the name drops nothing. - assert!(!leaves_a_drop_to_the_storage_keys( - &NativeTree::try_new(keeping_window_fields( - input_with_fields(1), - Arc::clone(&out), - 1 - )) - .unwrap(), - &named, - &single, - false, - )); - // A merge below the drop hands over one series per canonical label set, which is unique even - // where the grouping treats an empty label value as no label, so the key columns decide nothing. - assert!(!leaves_a_drop_to_the_storage_keys( - &NativeTree::try_new(window_on( - merge_over(input_on(Arc::clone(&out))), - Some(Arc::clone(&out)), - Arc::clone(&out), - )) - .unwrap(), - &named, - &single, - false, + &map_over(rewritten(plain, "dst", "host")), + &scan_tags() )); } @@ -2665,8 +2434,7 @@ fn a_drop_the_labels_prove_is_counted_only_when_its_keys_do_not() { #[test] fn a_rewrite_after_ranking_or_grouping_may_alias() { let out = grid(0, 60_000, 4); - // A name-keeping window, so the ranking below is the only node whose answer is in question. - let selected = || keeping_window_fields(input_with_fields(1), Arc::clone(&out), 1); + let selected = || window_fields(input_with_fields(1), Arc::clone(&out), 1); let vector = |op| NativeNode { expr: NativeExpr::Vector { input: Box::new(selected()), @@ -2707,10 +2475,7 @@ fn a_rewrite_after_ranking_or_grouping_may_alias() { ), ("sum by host", grouped_sum), ] { - assert!( - !duplicates_canonical_labels(&inner, &source_tags, &[], true), - "{name}" - ); + assert!(!duplicates_canonical_labels(&inner, &source_tags), "{name}"); let output = inner.output.clone(); let renamed = NativeNode { expr: NativeExpr::Map { @@ -2726,7 +2491,7 @@ fn a_rewrite_after_ranking_or_grouping_may_alias() { }; assert!(renamed.validate().is_ok(), "{name}"); assert!( - duplicates_canonical_labels(&renamed, &source_tags, &[], true), + duplicates_canonical_labels(&renamed, &source_tags), "{name}" ); } @@ -2846,9 +2611,7 @@ fn a_merge_answers_one_series_per_set() { ); assert!(!duplicates_canonical_labels( &merge_over(aliasing), - &scan_tags(), - &[], - true + &scan_tags() )); } @@ -2898,64 +2661,25 @@ fn a_window_unifies_its_own_contributions() { ] { assert_eq!( child_answer, - unifies_its_own_contributions(&child, &[], &[]), + unifies_its_own_contributions(&child), "{name}: the child's own answer" ); assert_eq!( window_answer, - unifies_its_own_contributions(&window_of(child, RangeFunction::LastOverTime), &[], &[]), + unifies_its_own_contributions(&window_of(child, RangeFunction::LastOverTime)), "{name}: a name-keeping window unifies whatever its child reported, and only a \ boundary leaves it aliasing" ); } - // A function that drops the metadata labels can give two series one label set, so the window - // reports it whatever its child said - and a caller that cannot say which metric the child reads - // has no proof to offer on the child's behalf. - for child in [collected(), fragments()] { + // A function that drops the metric name can give two series one label set whatever its child + // answered, so the window reports it whatever the child said. + for child in [collected(), fragments(), boundary()] { assert_eq!( Some(false), - unifies_its_own_contributions(&window_of(child, RangeFunction::Rate), &[], &[]), - "a name-dropping window over an unproved source may alias" + unifies_its_own_contributions(&window_of(child, RangeFunction::Rate)), + "a name-dropping window may alias" ); } - // A boundary is the one child the proof says nothing about: the window folds the pieces of one - // series one at a time, so what it emits aliases them whatever the labels say. - assert_eq!( - Some(false), - unifies_its_own_contributions(&window_of(boundary(), RangeFunction::Rate), &[], &[]), - "a boundary leaves the window's own answer aliasing" - ); - // The proof is the one the canonical pass reads: a child whose labels lost the name, over a - // source bound to a single metric without a metadata tag of its own, keeps the name-dropping - // window from reporting fragments. - let named = [vec!["host".to_string(), "__name__".to_string()]]; - let single = [Some(Arc::::from("some_metric"))]; - let dropped = map_over(input_on(Arc::clone(&out))); - assert_eq!( - Some(true), - unifies_its_own_contributions( - &window_of(dropped.clone(), RangeFunction::Rate), - &named, - &single - ), - "a single metric's dropped name tells none of its series apart" - ); - assert_eq!( - Some(false), - unifies_its_own_contributions( - &window_of(dropped.clone(), RangeFunction::Rate), - &named, - &[None] - ), - "a source that may hand over several metrics is not proved" - ); - let mut typed = named.clone(); - typed[0].push("__type__".to_string()); - assert_eq!( - Some(false), - unifies_its_own_contributions(&window_of(dropped, RangeFunction::Rate), &typed, &single), - "a metadata tag the window drops is not proved constant" - ); } /// `info` reads its info side through a canonical merge: the merge only collects the pieces of one diff --git a/src/promql/src/native/runs.rs b/src/promql/src/native/runs.rs new file mode 100644 index 00000000000..5322221ffab --- /dev/null +++ b/src/promql/src/native/runs.rs @@ -0,0 +1,143 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The grid positions a field occupies, as sorted, disjoint runs. +//! +//! A node that rewrites labels — a `Map` dropping the metric name, a `Window` dropping the name and +//! the empty-valued labels — can turn two input series into one label set, and every layer above +//! then reads the two chunks as one series. What decides whether that pair is legal is which +//! positions of each field already hold a value: two values of one field at one position cannot be +//! merged away, while a chunk that fills positions the set left empty merges cleanly, as +//! [`crate::grid_series::merge_same_labelset`] does. +//! +//! The native runtime's partition-local [`crate::native::execute::Emitted`] table keeps this state +//! per label set and field as runs, so a chunk of many positions costs one run rather than one +//! entry per position. + +/// One run of grid positions, half-open, that hold a value in one field. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Run { + pub(crate) start: usize, + pub(crate) end: usize, +} + +impl Run { + /// Whether two runs share a position. + pub(crate) fn overlaps(self, other: Self) -> bool { + self.start < other.end && other.start < self.end + } +} + +// How many times `absorb` rebuilt the stored runs instead of appending to them. +// +// Test-only, and per thread: the runs stored afterwards are the same either way, so a rebuild shows +// only in what it costs, and a test that reads its own thread's figure is not disturbed by another +// test's chunks. +#[cfg(test)] +thread_local! { + static ABSORB_REBUILDS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// How many times this thread's [`absorb`] rebuilt the stored runs. +#[cfg(test)] +pub(crate) fn absorb_rebuilds() -> usize { + ABSORB_REBUILDS.with(std::cell::Cell::get) +} + +/// Merges `incoming` into `stored`, keeping the result in position order and joining the runs that +/// touch. +/// +/// Within a chunk one pass over both sides is what keeps this linear, since each side is sorted and +/// disjoint. Adding the runs one at a time and rebuilding the array after each would cost the square +/// of the runs a chunk yields: a chunk of one hundred thousand positions alternating between values +/// and nulls yields fifty thousand runs, and re-merging after each is over a billion element moves +/// for a single series. +/// +/// One pass per chunk is still a pass over everything the series has stored so far, so it is not +/// free once the *history* grows across chunks: a series split into `K` chunks of one position each, +/// a hole apart, stores one more run per chunk and moves `K(K+1)/2` of them, with no label collision +/// and no illegal input anywhere. A chunk that starts no earlier than the end of the last stored run +/// only ever adds to the end — the stored runs are sorted and disjoint, so every one of them ends +/// before anything this chunk holds — and it is appended instead, which is amortised constant per +/// run. That is the shape a scan hands over as its positions move forward chunk after chunk. A chunk +/// that starts before the last stored run can split a run or reach back past it, so it still costs +/// the full merge. +/// +/// Either way the result is sorted, disjoint, and has the runs that touch joined, which is what a +/// look-up by [`Run::overlaps`] on the runs stored here relies on. +pub(crate) fn absorb(stored: &mut Vec, incoming: &[Run]) { + if incoming.is_empty() { + return; + } + if stored + .last() + .is_none_or(|last| incoming[0].start >= last.end) + { + append_runs(stored, incoming); + } else { + merge_runs(stored, incoming); + } +} + +/// Appends `incoming` to `stored`, all of which lies past the last stored run. +/// +/// Nothing already stored has to be placed again: the two sides are each in position order and +/// nothing `incoming` holds can reach back into what is stored, so its runs are pushed on the end and +/// the vector's own growth is the only move — amortised over the runs appended. Where its first run +/// touches the end of the last stored run the two are one run, as they are after a merge, so it is +/// joined rather than left beside it. +fn append_runs(stored: &mut Vec, incoming: &[Run]) { + let mut incoming = incoming; + if let Some(last) = stored.last_mut() + && let Some(first) = incoming.first() + && first.start <= last.end + { + last.end = last.end.max(first.end); + incoming = &incoming[1..]; + } + stored.extend_from_slice(incoming); +} + +/// Merges `incoming` into `stored`, where the two can interleave. +/// +/// One pass is enough because each side is sorted and disjoint: whichever run starts first is +/// placed, and it joins the run before it where the two touch. +fn merge_runs(stored: &mut Vec, incoming: &[Run]) { + #[cfg(test)] + ABSORB_REBUILDS.with(|rebuilds| rebuilds.set(rebuilds.get() + 1)); + let mut merged: Vec = Vec::with_capacity(stored.len() + incoming.len()); + let (mut a, mut b) = (0, 0); + while a < stored.len() || b < incoming.len() { + let larger = match (stored.get(a), incoming.get(b)) { + (Some(left), Some(right)) if right.start < left.start => { + b += 1; + *right + } + (Some(left), _) => { + a += 1; + *left + } + (None, Some(right)) => { + b += 1; + *right + } + (None, None) => break, + }; + match merged.last_mut() { + Some(last) if larger.start <= last.end => last.end = last.end.max(larger.end), + _ => merged.push(larger), + } + } + *stored = merged; +} diff --git a/src/promql/src/native/segment.rs b/src/promql/src/native/segment.rs index 6a32984055d..d3dd151ad19 100644 --- a/src/promql/src/native/segment.rs +++ b/src/promql/src/native/segment.rs @@ -268,6 +268,21 @@ fn cut_as_segment(node: &NativeNode, canonical_segmentation: bool) -> bool { !sources.is_empty() } +/// Whether any canonical merge in `node` answers with histogram samples, given the kind its input +/// carries. +/// +/// A merge collects the pieces of one label set and hands on the samples it was given, so its answer +/// kind is its child's. A histogram answer standing on a merge would travel through the canonical +/// boundary carrying histogram frames, a pairing no execution exercises yet: the planner keeps such +/// a statement on the existing plan until that pairing is covered, and it has to look at every +/// merge, not only a root one. +pub fn merge_answers_histogram(node: &NativeNode, input_kind: FieldKind) -> bool { + node.any(&|node| { + matches!(node.expr, NativeExpr::Merge { .. }) + && node.output_field_kind(Some(input_kind)) == Some(FieldKind::Histogram) + }) +} + /// Whether the merge producer's entire prefix can run on each scan partition independently. pub fn canonical_local_prefix(node: &NativeNode) -> bool { match &node.expr { diff --git a/src/promql/src/range_eval.rs b/src/promql/src/range_eval.rs index e0223b4ac58..00ef42ddf67 100644 --- a/src/promql/src/range_eval.rs +++ b/src/promql/src/range_eval.rs @@ -200,36 +200,12 @@ pub struct RangeEvaluator { batch: Option, /// Windows accumulated for the next batch call. pending: PendingBatch, - /// Buffers the bulk ingest shifts and compacts a chunk into, kept between chunks so a series - /// that needs them does not allocate per push. - bulk_scratch: BulkScratch, /// Batch kernel calls made so far. Test-only: both dispatch shapes produce the same output, /// so a test comparing only output could not tell which one ran. #[cfg(test)] batch_calls: usize, } -/// The buffers the bulk ingest needs for a chunk whose samples are not retained as they arrive: -/// the timestamps shifted by the selector offset, and one compacted column per field. -#[derive(Default)] -struct BulkScratch { - timestamps: Vec, - columns: Vec>, -} - -impl BulkScratch { - /// The bytes these buffers hold, by capacity. - fn bytes(&self) -> usize { - self.timestamps.capacity() * std::mem::size_of::() - + self.columns.capacity() * std::mem::size_of::>() - + self - .columns - .iter() - .map(|column| column.capacity() * std::mem::size_of::()) - .sum::() - } -} - /// Windows accumulated for the next batch call: one list per field. struct PendingBatch { /// Grid index of each accumulated point, ascending. @@ -710,20 +686,11 @@ impl RangeEvaluator { bulk_ingests: 0, batch: None, pending: PendingBatch::with_fields(input.fields.len()), - bulk_scratch: BulkScratch::default(), #[cfg(test)] batch_calls: 0, }) } - /// The evaluation grid this evaluator decides points of. - /// - /// Every window ends at a point of it and is read at that point's timestamp, so a caller - /// reserving for what the evaluator is about to allocate reserves this grid's timestamps. - pub fn grid(&self) -> &Arc { - &self.grid - } - /// Chunks appended without the per-sample loop. Test-only: both paths produce the same /// output by construction, so a test that only compared output could not tell whether the /// bulk path ran at all. @@ -749,9 +716,8 @@ impl RangeEvaluator { /// Counted by capacity, so it is what the storage holds rather than what it uses: the retained /// samples per field (timestamps, values, the reset index and the native histograms with the /// payload each of them owns), the row a push fills with the histograms it boxes, the identity - /// of the open series, the pending batch's own vectors and the buffers the bulk ingest shifts - /// and compacts into. Left out are the kernel's scratch buffers, this struct's `Vec` headers - /// and the grid, which every chunk addressed on it shares. + /// of the open series and the pending batch's own vectors. Left out are the kernel's scratch + /// buffers, this struct's `Vec` headers and the grid, which every chunk addressed on it shares. /// /// The figure is an estimate, not a ledger of every allocation: a histogram's payload is /// measured once when it is admitted and released with it, so neither the retained samples nor @@ -769,7 +735,6 @@ impl RangeEvaluator { + boxed_bytes(&self.row) + self.series.as_ref().map_or(0, SeriesState::identity_bytes) + self.pending.bytes() - + self.bulk_scratch.bytes() } /// An upper bound of the rows the next push hands out, for input whose last sample is at @@ -1101,139 +1066,63 @@ impl RangeEvaluator { Ok(()) } - /// Appends the chunk in bulk when every field is an entirely present float column. + /// Appends the chunk in bulk when every field is an entirely present, unshifted float column. /// - /// Returns `false` when a condition of the per-sample path applies, so the caller can run it. - /// Only the shapes that path alone can read do: a field holding native histogram samples or - /// mixed floats and histograms -- `dense()` reports [`DenseSamplesRef::Mixed`] for the latter, - /// keeping its windows on the histogram path where the mixed-window rules decide the result -- - /// and a column with missing samples, which are absent rows rather than values. + /// Returns `false` when any condition of the per-sample path applies, so the caller can run + /// it. A chunk carrying a stale marker takes that path too: the marker drops the whole row, + /// and reproducing that here would cost the same scan the loop already makes. /// - /// A selector offset and a stale marker do not: the shift and the rows the marker drops are - /// applied here, over the columns, instead of handing the chunk to the per-sample loop. The - /// search for a marker is one short-circuiting pass, the same pass the loop would have made. + /// A field mixing floats and native histograms never qualifies: `dense()` reports + /// [`DenseSamplesRef::Mixed`] for it, keeping its windows on the histogram path where the + /// mixed-window rules decide the result. fn ingest_in_bulk( &mut self, timestamps: &[i64], fields: &SeriesFields, drop_stale: bool, - ) -> Result { + ) -> bool { // Lengths are checked by `check_chunk` for raw input and by `push_series` for evaluated // input, so only the shape of the columns is decided here. + if self.window.offset_ms != 0 { + return false; + } let mut columns = Vec::with_capacity(fields.len()); for field in fields.iter() { let Some(DenseSamplesRef::Float(values)) = field.dense() else { - return Ok(false); + return false; }; if values.null_count() != 0 { - return Ok(false); + return false; } - columns.push(values.values().as_ref()); + columns.push(values); } - // A stale marker drops its whole row, so which rows go is decided before anything is - // retained. A chunk holding none -- the common case -- is judged by one scan that stops at - // the first marker and then leaves the columns as they are. - let stale_rows = if drop_stale - && columns - .iter() - .any(|values| values.iter().any(|value| is_prometheus_stale_nan(*value))) + // A kept sample's shifted time must fit `i64`; with no offset it is the source timestamp, + // which `RawTimestamps` already validated as increasing and which the frame widened into + // this array without narrowing, so there is nothing left to check here. + if drop_stale + && columns.iter().any(|values| { + values + .values() + .iter() + .any(|value| is_prometheus_stale_nan(*value)) + }) { - let mut stale = vec![false; timestamps.len()]; - for values in &columns { - for (row, dropped) in stale.iter_mut().enumerate() { - *dropped |= is_prometheus_stale_nan(values[row]); - } - } - Some(stale) - } else { - None - }; - self.append_float_chunk(timestamps, &columns, stale_rows.as_deref())?; + return false; + } + for (field, values) in self.fields.iter_mut().zip(columns) { + field.push_bulk(timestamps, values.values()); + } #[cfg(test)] { self.bulk_ingests += 1; } - Ok(true) - } - - /// Retains one chunk of entirely present float columns. - /// - /// `stale` marks the rows a stale marker drops, `None` when the chunk holds none. The retained - /// samples are the chunk's rows less those, on the evaluation timeline: the selector offset - /// shifts them, and only a chunk that needs neither keeps the incoming timestamps as they are. - fn append_float_chunk( - &mut self, - timestamps: &[i64], - columns: &[&[f64]], - stale: Option<&[bool]>, - ) -> Result<()> { - let offset_ms = self.window.offset_ms; - if offset_ms == 0 && stale.is_none() { - // The source timestamps, which `RawTimestamps` already validated as increasing and - // which the frame widened into this array without narrowing: nothing to shift, nothing - // to check, and the two appends become two `memcpy`s. - for (field, values) in self.fields.iter_mut().zip(columns) { - field.push_bulk(timestamps, values); - } - return Ok(()); - } - let BulkScratch { - timestamps: mut shifted, - columns: mut compacted, - } = std::mem::take(&mut self.bulk_scratch); - shifted.clear(); - shifted.reserve(timestamps.len()); - for (row, ×tamp) in timestamps.iter().enumerate() { - if stale.is_some_and(|stale| stale[row]) { - continue; - } - // Like RangeManipulate, any kept sample whose shifted time leaves i64 fails the series, - // whichever path retains it. - let Some(shifted_timestamp) = timestamp.checked_add(offset_ms) else { - self.bulk_scratch = BulkScratch { - timestamps: shifted, - columns: compacted, - }; - return Err(DataFusionError::Execution(format!( - "sample timestamp {timestamp} overflows with offset {offset_ms} ms" - ))); - }; - shifted.push(shifted_timestamp); - } - match stale { - None => { - for (field, values) in self.fields.iter_mut().zip(columns) { - field.push_bulk(&shifted, values); - } - } - Some(stale) => { - compacted.resize_with(columns.len(), Vec::new); - for (retained, values) in compacted.iter_mut().zip(columns) { - retained.clear(); - retained.extend( - values - .iter() - .enumerate() - .filter(|(row, _)| !stale[*row]) - .map(|(_, &value)| value), - ); - } - for (field, values) in self.fields.iter_mut().zip(&compacted) { - field.push_bulk(&shifted, values); - } - } - } - self.bulk_scratch = BulkScratch { - timestamps: shifted, - columns: compacted, - }; - Ok(()) + true } /// Retains the samples of `fields` at `timestamps`. From `raw` input a row holding a stale /// marker in any field is dropped; evaluated input keeps it as a value. fn ingest(&mut self, timestamps: &[i64], fields: &SeriesFields, raw: bool) -> Result<()> { - if self.ingest_in_bulk(timestamps, fields, raw)? { + if self.ingest_in_bulk(timestamps, fields, raw) { return Ok(()); } let mut samples: Vec<_> = fields.iter().map(|field| field.iter()).collect(); diff --git a/src/promql/src/range_eval/function.rs b/src/promql/src/range_eval/function.rs index db8ea8d3782..ce54432ed28 100644 --- a/src/promql/src/range_eval/function.rs +++ b/src/promql/src/range_eval/function.rs @@ -18,28 +18,10 @@ //! operation order. Arithmetic that GreptimeDB's range UDFs compute the same way is shared with //! them. -use std::collections::VecDeque; -use std::ops::Range; - use crate::functions::{EdgeKind, count_edges, extremum, instant_delta, kahan_inc}; use crate::go_sort; use crate::range_eval::{BatchKernel, FloatWindow, FloatWindows, RangeModifier, WindowKernel}; -/// The samples the batch kernels examine, counted per value. -/// -/// A batch kernel exists so a field is walked once per batch instead of once per window, which the -/// results cannot show: both shapes compute the same values. Counting what a batch kernel touches -/// is what tells them apart. -#[cfg(test)] -pub(super) static SAMPLES_EXAMINED: std::sync::atomic::AtomicU64 = - std::sync::atomic::AtomicU64::new(0); - -/// Counts `count` samples examined by a batch kernel; a no-op outside tests. -fn examined(_count: usize) { - #[cfg(test)] - SAMPLES_EXAMINED.fetch_add(_count as u64, std::sync::atomic::Ordering::Relaxed); -} - /// A PromQL range-vector function over float and native histogram samples. #[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] pub enum RangeFunction { @@ -109,34 +91,28 @@ impl RangeFunction { /// Whether the evaluator has to index counter resets for this function. /// - /// This must hold for exactly the functions that read the index: `rate` and `increase`, whose - /// kernels read [`FloatWindow::counter_resets`] -- the same as saying `IS_COUNTER` in - /// `extrapolated_rate` -- and `resets`, whose batch kernel reads [`FloatWindows::resets_of`]. - /// A kernel that corrects or counts counters without the index being maintained would silently - /// miss every reset, so a new reset-reading function has to be added here as well. + /// This must hold for exactly the kernels that read [`FloatWindow::counter_resets`], which is + /// the same as saying `IS_COUNTER` in `extrapolated_rate`: `rate` and `increase`. A kernel + /// that corrects counters without the index being maintained would silently drop every + /// correction, so a new counter function has to be added here as well. /// - /// It governs both readers of the index. When it returns `false`, `resets` stays empty, - /// `resets_of` yields nothing and `counter_resets` is empty, so a kernel reading either would - /// see a series that never reset. + /// It governs both readers of the index: the per-window kernel above, and the batch kernel, + /// which reads [`FloatWindows::resets_of`]. When it returns `false`, `resets` stays empty and + /// `resets_of` yields nothing, so a batch kernel for a counter function would skip every + /// correction and report `delta`-like values. pub(crate) fn indexes_counter_resets(self) -> bool { - matches!(self, Self::Rate | Self::Increase | Self::Resets) + matches!(self, Self::Rate | Self::Increase) } /// The batch form of this function, when it has one. /// - /// A batch form exists for the functions whose window answer is a running summary of the field - /// -- the two endpoints and the resets for the counter rates, a movement of the window's bounds - /// for the extremes, the resets inside it, and the change edges inside it. The rest keep the - /// per-window form, which the evaluator falls back to, so adding a function here is what - /// changes which shape its evaluation takes. + /// Only the counter functions have one. The rest keep the per-window form, which the + /// evaluator falls back to, so adding a function here is what changes which shape its + /// evaluation takes. pub(crate) fn batch_kernel(self) -> Option { match self { Self::Rate => Some(Box::new(extrapolated_rate_batch::)), Self::Increase => Some(Box::new(extrapolated_rate_batch::)), - Self::MinOverTime => Some(extremum_batch::()), - Self::MaxOverTime => Some(extremum_batch::()), - Self::Resets => Some(Box::new(resets_batch)), - Self::Changes => Some(changes_batch()), _ => None, } } @@ -512,189 +488,6 @@ pub(super) struct ExtrapolationSpan { pub samples_minus_one: usize, } -/// The batch form of `min_over_time` (`IS_MIN`) and `max_over_time`: one pass over the union of -/// the batch's windows, carrying the candidates that can still become the window's extremum. -/// -/// The windows of a batch start and end no earlier than the ones before them, so a sample that -/// lost to a later, strictly better one can never win a later window either: it leaves the back of -/// the queue, and a sample the window start has passed leaves the front. That is exactly what -/// [`extremum`] computes window by window, which is why the deque keeps the arrival order of equal -/// values (and with it the sign of tied zeros) and why a NaN never wins while the window holds -/// anything else. -fn extremum_batch() -> BatchKernel { - let mut candidates = SlidingExtremum::::default(); - Box::new( - move |batch: &FloatWindows<'_>, out: &mut Vec>| { - out.clear(); - out.reserve(batch.windows.len()); - // Batches are independent: the retained arrays of the previous one may have been drained - // since, and the samples behind the first window are gone anyway. - candidates.clear(); - let values = batch.values; - let sliding = pays_to_slide(batch.windows); - for span in batch.windows { - let (start, end) = (span.start, span.end); - out.push(if sliding { - candidates.evaluate(values, start, end) - } else if start == end { - None - } else { - examined(end - start); - extremum(values[start..end].iter().copied(), IS_MIN) - }); - } - }, - ) -} - -/// The samples between the first window's start and the last window's end: what a window-moving -/// summary of a batch has to walk, against the sum of the windows' lengths a rescan walks. -fn union_len(windows: &[Range]) -> usize { - match (windows.first(), windows.last()) { - (Some(first), Some(last)) => last.end.saturating_sub(first.start), - _ => 0, - } -} - -/// Whether walking the union of a batch's windows once beats scanning each window on its own. -/// -/// Reuse pays off in proportion to how much consecutive windows overlap: a window-moving summary -/// visits every sample between the first window's start and the last one's end -- plus a step per -/// window -- while a scan visits every sample of every window. On wide steps that barely overlap -/// the walk is the more expensive of the two. -fn pays_to_slide(windows: &[Range]) -> bool { - let scanned: usize = windows.iter().map(|span| span.len()).sum(); - union_len(windows) + windows.len() < scanned -} - -/// The extrema candidates of one window, carried to the next. -/// -/// Consecutive windows ascend, so the queue is only trimmed at the front and appended at the back. -#[derive(Default)] -struct SlidingExtremum { - /// Indices of the samples that can still become the extremum, in arrival order. A strictly - /// better sample evicts the ones queued before it, so the front is the window's extremum. - candidates: VecDeque, - /// Backs the all-NaN window, which reports the last NaN it holds, as [`extremum`] does. - latest_nan: Option, - /// The window the queue describes; `None` until the first window of a batch. - previous: Option<(usize, usize)>, -} - -impl SlidingExtremum { - fn clear(&mut self) { - self.candidates.clear(); - self.latest_nan = None; - self.previous = None; - } - - /// Whether `value` replaces `current`, the comparison [`extremum`] replaces on. - #[inline] - fn beats(value: f64, current: f64) -> bool { - if IS_MIN { - value < current - } else { - value > current - } - } - - /// The extremum of `values[start..end]`, `None` when the window holds no sample. - fn evaluate(&mut self, values: &[f64], start: usize, end: usize) -> Option { - let append_start = match self.previous { - Some((previous_start, previous_end)) - if start >= previous_start && end >= previous_end => - { - while self.candidates.front().is_some_and(|&index| index < start) { - self.candidates.pop_front(); - } - if self.latest_nan.is_some_and(|index| index < start) { - self.latest_nan = None; - } - previous_end.max(start) - } - // A retreating window would need samples the queue no longer holds. A batch's windows - // ascend, so this is the state a call starts from, not a case the walk reaches. - Some(_) | None => { - self.clear(); - start - } - }; - for (offset, &value) in values[append_start..end].iter().enumerate() { - if value.is_nan() { - self.latest_nan = Some(append_start + offset); - continue; - } - while let Some(&back) = self.candidates.back() { - if !Self::beats(value, values[back]) { - break; - } - self.candidates.pop_back(); - } - self.candidates.push_back(append_start + offset); - } - examined(end.saturating_sub(append_start)); - self.previous = Some((start, end)); - self.candidates - .front() - .or(self.latest_nan.as_ref()) - .map(|&index| values[index]) - } -} - -/// The batch form of `changes`: a prefix count of the change edges between neighbouring samples, -/// so a window's answer is the difference of two entries instead of a scan. -/// -/// The edges are counted with [`count_edges`], the predicate the per-window kernel uses, so the -/// two shapes agree on what a change is -- including that two NaNs, whose comparison is always -/// false, are not one. -fn changes_batch() -> BatchKernel { - let mut prefix: Vec = Vec::new(); - Box::new( - move |batch: &FloatWindows<'_>, out: &mut Vec>| { - out.clear(); - out.reserve(batch.windows.len()); - let Some((first, last)) = batch.windows.first().zip(batch.windows.last()) else { - return; - }; - let (union_start, union_end) = (first.start, last.end); - let scan = !pays_to_slide(batch.windows); - // `prefix[i]` counts the change edges among `values[union_start..=i]`, so the window - // `[start, end)` counts the edges inside it: its first one is the edge into `start + 1`. - prefix.clear(); - if !scan { - prefix.push(0); - examined(union_end.saturating_sub(union_start)); - for index in union_start + 1..union_end { - let edge = count_edges(&batch.values[index - 1..=index], EdgeKind::Changes); - prefix.push(prefix[prefix.len() - 1] + edge); - } - } - for span in batch.windows { - out.push((span.start != span.end).then(|| { - if scan { - examined(span.len()); - return count_edges(&batch.values[span.start..span.end], EdgeKind::Changes) - as f64; - } - let below = prefix[span.start - union_start]; - let inside = prefix[span.end - 1 - union_start]; - (inside - below) as f64 - })); - } - }, - ) -} - -/// The batch form of `resets`: the resets inside each window, read off the evaluator's reset -/// index. No sample is scanned: a window's answer is a count of index entries between its bounds. -fn resets_batch(batch: &FloatWindows<'_>, out: &mut Vec>) { - out.clear(); - out.reserve(batch.windows.len()); - for (index, span) in batch.windows.iter().enumerate() { - out.push((span.start != span.end).then(|| batch.resets_of(index).len() as f64)); - } -} - /// The factor `extrapolatedRate` scales the sampled increase by. `zero_point` is the first value /// over the increase, given only when Prometheus extrapolates a counter to zero: it is the share of /// the sampled interval that lies before `span.sampled_from`, so `0` says the counter's zero point diff --git a/src/promql/src/range_eval/histogram.rs b/src/promql/src/range_eval/histogram.rs index 196d8e2bb33..3842226db82 100644 --- a/src/promql/src/range_eval/histogram.rs +++ b/src/promql/src/range_eval/histogram.rs @@ -159,8 +159,7 @@ pub(super) fn evaluate( HistogramAggregate::Avg }; // The hints are read from the window's own histograms, so nothing is cloned for the - // check; the fold below reads them by reference too and copies only the accumulator - // it starts from. + // check; the fold below clones one sample at a time as it consumes them. if has_contradictory_reset_hints(histograms.iter().map(|(_, h)| h)) { record_warning( collector, @@ -174,7 +173,7 @@ pub(super) fn evaluate( // flag is the reconciliation its own adds already observed. Re-walking the pairs to // learn either would double the histogram work of every window. let (result, custom_reconciled) = - sum_or_avg(op, histograms.iter().map(|(_, h)| Ok(Cow::Borrowed(h))))?; + sum_or_avg(op, histograms.iter().map(|(_, h)| Ok(h.clone())))?; if result.is_none() { record_warning( collector, diff --git a/src/promql/src/range_eval/tests.rs b/src/promql/src/range_eval/tests.rs index 423fea39951..de6d3fcf269 100644 --- a/src/promql/src/range_eval/tests.rs +++ b/src/promql/src/range_eval/tests.rs @@ -38,7 +38,6 @@ use datatypes::series::{Label, SeriesLabels}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; -use super::function::SAMPLES_EXAMINED; use super::{ CounterReset, FieldSamples, FloatWindow, POSITION_COMPARISONS, PendingBatch, RangeEvaluator, RangeFunction, RangeModifier, RangeWindow, WindowKernel, accumulate, subquery_grid, @@ -1924,237 +1923,6 @@ fn a_function_without_a_batch_form_stays_on_the_per_window_path() { assert_eq!(evaluator.batch_call_count(), 0); } -/// Rows of a counter that resets twice, plateaus once, steps unevenly and holds one stale marker: -/// every shape an extremum, a reset count or a change count can get wrong at a window bound. -fn summary_rows(points: usize) -> Vec { - let mut value = 0.0; - (0..points) - .map(|index| { - value = match index { - 200 | 500 => 1.0, - 300 => value, - _ if index % 3 == 0 => value + 2.0, - _ => value + 1.0, - }; - let sample = if index == 150 { stale() } else { value }; - (index as i64 * 10_000, vec![Some(sample)]) - }) - .collect() -} - -/// The functions whose window answer is a running summary of the field -- the extremes, the resets -/// and the changes -- reach the batch kernel, and its answers equal the per-window ones. -/// -/// The series is longer than `EVALUATION_BATCH`, so the batch also flushes in the middle of the -/// evaluation and has to carry the summary across the flush. A summary read off the wrong window -/// bounds, or a reset counted across one, differs from the per-window result on these rows. -#[test] -fn window_summaries_reach_the_batch_kernel() { - let points = 700; - let rows = summary_rows(points); - let window = RangeWindow { - range_ms: 40_000, - offset_ms: 0, - anchor_ms: None, - modifier: None, - }; - for function in [ - RangeFunction::MinOverTime, - RangeFunction::MaxOverTime, - RangeFunction::Resets, - RangeFunction::Changes, - ] { - let mut batch = RangeEvaluator::try_new( - &raw_schema(1), - grid(0, 10_000, points as u64), - window, - function, - ) - .unwrap(); - // The per-window path is the same evaluator without the batch kernel, which leaves the - // rest of the evaluation identical. - let mut per_window = RangeEvaluator::try_new( - &raw_schema(1), - grid(0, 10_000, points as u64), - window, - function, - ) - .unwrap(); - per_window.batch = None; - - let piece = chunk(&rows, 1, 0, true); - let batched = collect_chunks(&mut batch, std::slice::from_ref(&piece)); - let expected = collect_chunks(&mut per_window, std::slice::from_ref(&piece)); - assert!( - batch.batch_call_count() > 0, - "{function:?} never reached its batch kernel" - ); - assert_same_series( - fold_chunks(&batched, batch.output_schema()).as_ref(), - fold_chunks(&expected, per_window.output_schema()).as_ref(), - &format!("{function:?}"), - ); - } -} - -/// A batch kernel walks a field once per batch, not once per grid point. -/// -/// The per-window form visits every sample of every window, so `points` windows of `width` samples -/// visit `points * width`. A batch visits the union of its windows once plus one entry per window, -/// so the total stays close to the samples the series holds whatever the width is. The figure is -/// counted rather than timed, which is what makes it a property of the code and not of the -/// machine. -#[test] -fn window_summaries_are_not_rescanned_per_step() { - let points = 2_048u64; - let width = 60; - let rows = summary_rows(points as usize); - let window = RangeWindow { - range_ms: width as i64 * 10_000, - offset_ms: 0, - anchor_ms: None, - modifier: None, - }; - // What rescanning each window would visit, and what a walk of the retained samples costs. - let rescanned = points * width; - let once_over = points * 2; - let examined = |function| -> u64 { - SAMPLES_EXAMINED.store(0, std::sync::atomic::Ordering::Relaxed); - let mut evaluator = - RangeEvaluator::try_new(&raw_schema(1), grid(0, 10_000, points), window, function) - .unwrap(); - evaluator.push(&chunk(&rows, 1, 0, true)).unwrap(); - SAMPLES_EXAMINED.load(std::sync::atomic::Ordering::Relaxed) - }; - for function in [ - RangeFunction::MinOverTime, - RangeFunction::MaxOverTime, - RangeFunction::Changes, - ] { - let visits = examined(function); - assert!( - visits > 0, - "{function:?} reached no batch kernel, so nothing was counted" - ); - assert!( - visits <= once_over, - "{function:?} examined {visits} samples over {points} grid points, more than the \ - {once_over} a single walk of the series costs: the windows are being rescanned" - ); - } - // `resets` reads the reset index, so it touches no sample at all. - assert_eq!( - examined(RangeFunction::Resets), - 0, - "resets scanned samples instead of reading the reset index" - ); - assert!(rescanned > once_over * 10, "the comparison proves nothing"); -} - -/// The samples of a chunk kept in segments, which `dense()` refuses: the per-sample loop is the -/// only path that retains them, so one chunk can be handed to either path unchanged. -fn segmented(piece: &RawSeriesChunk, split: usize) -> RawSeriesChunk { - let mut piece = piece.clone(); - let len = piece.time.len(); - let fields = piece - .fields - .iter() - .map(|field| { - Samples::try_from_chunks(vec![ - field.slice(0, split).unwrap(), - field.slice(split, len - split).unwrap(), - ]) - .unwrap() - }) - .collect(); - piece.fields = SeriesFields::try_from_vec(fields).unwrap(); - piece -} - -/// A chunk read with a selector offset is appended in bulk, with the shift applied there rather -/// than by handing the whole chunk to the per-sample loop. -/// -/// The same rows go to two evaluators, one from a single dense chunk -- which the bulk path takes -/// -- and one from a chunk kept in segments, which only the per-sample loop retains. The results -/// have to be identical, and only the first may count a bulk ingest. -#[test] -fn an_offset_chunk_is_appended_in_bulk() { - let rows: Vec = (0..5) - .map(|index| (index as i64 * 10_000, vec![Some(index as f64)])) - .collect(); - let window = RangeWindow { - range_ms: 20_000, - offset_ms: 10_000, - anchor_ms: None, - modifier: None, - }; - let evaluate = |piece: &RawSeriesChunk| { - let mut evaluator = RangeEvaluator::try_new( - &raw_schema(1), - grid(0, 10_000, 5), - window, - RangeFunction::SumOverTime, - ) - .unwrap(); - let chunks = collect_chunks(&mut evaluator, std::slice::from_ref(piece)); - ( - fold_chunks(&chunks, evaluator.output_schema()), - evaluator.bulk_ingest_count(), - ) - }; - let piece = chunk(&rows, 1, 0, true); - let (bulk, bulk_ingests) = evaluate(&piece); - let (per_sample, per_sample_ingests) = evaluate(&segmented(&piece, 2)); - assert_eq!( - bulk_ingests, 1, - "a chunk shifted by an offset was handed to the per-sample loop" - ); - assert_eq!( - per_sample_ingests, 0, - "the segmented chunk took the bulk path, so the two were not compared" - ); - assert_same_series(bulk.as_ref(), per_sample.as_ref(), "offset"); -} - -/// A chunk holding a stale marker is appended in bulk too, with the rows the marker drops taken -/// out there, and the result is what retaining the same rows sample by sample produces. -#[test] -fn a_stale_marker_does_not_force_the_per_sample_loop() { - let mut rows: Vec = (0..5) - .map(|index| (index as i64 * 10_000, vec![Some(index as f64)])) - .collect(); - rows[2].1[0] = Some(stale()); - let window = RangeWindow { - range_ms: 20_000, - offset_ms: 0, - anchor_ms: None, - modifier: None, - }; - let evaluate = |piece: &RawSeriesChunk| { - let mut evaluator = RangeEvaluator::try_new( - &raw_schema(1), - grid(0, 10_000, 5), - window, - RangeFunction::LastOverTime, - ) - .unwrap(); - let chunks = collect_chunks(&mut evaluator, std::slice::from_ref(piece)); - ( - fold_chunks(&chunks, evaluator.output_schema()), - evaluator.bulk_ingest_count(), - ) - }; - let piece = chunk(&rows, 1, 0, true); - let (bulk, bulk_ingests) = evaluate(&piece); - let (per_sample, per_sample_ingests) = evaluate(&segmented(&piece, 2)); - assert_eq!( - bulk_ingests, 1, - "a chunk holding a stale marker was handed to the per-sample loop" - ); - assert_eq!(per_sample_ingests, 0); - assert_same_series(bulk.as_ref(), per_sample.as_ref(), "stale marker"); -} - #[test] fn empty_input_and_series_without_results() { let window = RangeWindow { diff --git a/src/promql/src/select.rs b/src/promql/src/select.rs index 50c66a33e16..1ffcc1cf9f4 100644 --- a/src/promql/src/select.rs +++ b/src/promql/src/select.rs @@ -32,7 +32,7 @@ use datatypes::series::time::{EvalGrid, GridRange, TimeAxis, union_positions}; use datatypes::series::{Label, SeriesLabels}; use crate::go_sort; -use crate::grid_series::{GridSeriesBuilder, external}; +use crate::grid_series::{GridSeriesBuilder, external, owned}; #[cfg(test)] mod tests; @@ -90,28 +90,17 @@ pub fn topk_at( // positions, not the whole grid. grid_points(schema, series)?; let mut order: Vec = (0..series.len()).collect(); - // The label pairs of a series are built once: a comparator that compared two metrics would - // rebuild both lists per comparison, and the groups read the same lists again. - let mut label_sets: Vec, Arc)>> = series - .iter() - .map(|item| labels_with_name(&item.metric)) - .collect(); - order.sort_by(|&a, &b| label_sets[a].cmp(&label_sets[b])); + order.sort_by(|&a, &b| compare_labels(&series[a].metric, &series[b].metric)); let mut groups: HashMap> = HashMap::new(); for index in order { groups - .entry(group_key_from(&label_sets[index], grouping)) + .entry(group_key(&series[index].metric, grouping)) .or_default() .push(index); } - // The groups hold what they need, so the label lists are freed before the point walk. - label_sets.clear(); let mut selected: Vec> = vec![Vec::new(); series.len()]; let mut candidates: Vec<(usize, usize, f64)> = Vec::new(); - // The values of one point, for the partial selection below: ranking them there leaves the - // candidates in member order, which is what hands ties to the earlier label set. - let mut values: Vec = Vec::new(); for members in groups.values() { // A point no member carries holds no candidate, so only the union of the members' // positions is visited: a hole in the grid never reaches the ranking. Each member reads @@ -133,41 +122,12 @@ pub fn topk_at( candidates.push((member, row, value)); } } + // Members are in label order, so a stable sort hands ties to the earlier label set. + candidates.sort_by(|a, b| rank(a.2, b.2, top)); let Some(k) = k_at(point)? else { continue; }; - // Which candidates win is the result; the order they were found in is not. The ranking - // is a partial selection of the k best values rather than an ordering of the whole - // group, and it runs on a copy of the values so that the candidates keep the member - // order the ties are resolved in. - let k = k.min(candidates.len()); - if k == 0 { - continue; - } - let boundary = (k < candidates.len()).then(|| { - values.clear(); - values.extend(candidates.iter().map(|candidate| candidate.2)); - values.select_nth_unstable_by(k, |a, b| rank(*a, *b, top)); - values[k] - }); - // The partition splits the run of values equal to its boundary, while ties go to the - // earlier label set: the ones equal to the boundary inside the front are the slots left - // for them, and the scan below fills those slots in member order, which is label order. - let mut tied = match boundary { - Some(boundary) => values[..k] - .iter() - .filter(|&&value| rank(value, boundary, top) == Ordering::Equal) - .count(), - None => 0, - }; - for &(member, row, value) in &candidates { - if let Some(boundary) = boundary { - match rank(value, boundary, top) { - Ordering::Less => {} - Ordering::Equal if tied > 0 => tied -= 1, - _ => continue, - } - } + for &(member, row, _) in candidates.iter().take(k) { selected[member].push((point, row)); } } @@ -258,9 +218,8 @@ pub fn limitk( // A series belongs to one group, whose points are visited in order. let mut cursor = series[index].fields.as_slice()[0].cursor(); for (point, row) in points { - // The sample is moved, not computed, so it goes straight into the result. let sample = cursor.at(row).expect("the axis and the fields are aligned"); - builder.push_refs(point, &[sample])?; + builder.push_samples(point, &[owned(sample)?])?; } result.extend(builder.finish(series[index].metric.clone())?); } @@ -281,7 +240,7 @@ pub fn limit_ratio(ratio: f64, schema: &SeriesSchema, series: &[Series]) -> Resu let ratio = ratio.clamp(-1.0, 1.0); let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone()))?; let mut result = Vec::new(); - let mut row: Vec> = Vec::with_capacity(schema.fields.len()); + let mut row = Vec::with_capacity(schema.fields.len()); for item in series { let offset = sample_offset(&item.metric); let kept = if ratio >= 0.0 { @@ -298,15 +257,14 @@ pub fn limit_ratio(ratio: f64, schema: &SeriesSchema, series: &[Series]) -> Resu item.fields.iter().map(|field| field.cursor()).collect(); for (point, index) in item.time.positions() { row.clear(); - // The samples are moved, not computed, so the row is read and pushed as it stands. for cursor in &mut cursors { - row.push( + row.push(owned( cursor .at(index) .expect("the axis and the fields are aligned"), - ); + )?); } - builder.push_refs(point, &row)?; + builder.push_samples(point, &row)?; } result.extend(builder.finish(item.metric.clone())?); } @@ -580,8 +538,6 @@ const GO_NAN: f64 = f64::from_bits(0x7FF8_0000_0000_0001); /// The order of `a` and `b` for `topk` (`top`) or `bottomk`; NaN ranks last for both. fn rank(a: f64, b: f64, top: bool) -> Ordering { - #[cfg(test)] - calls::record_rank(); match (a.is_nan(), b.is_nan()) { (true, true) => Ordering::Equal, (true, false) => Ordering::Greater, @@ -652,13 +608,8 @@ pub(crate) fn validate_topk_parameters(values: &[Option]) -> Result<()> { type GroupKey = Vec<(Arc, Arc)>; fn group_key(metric: &SeriesLabels, grouping: &Grouping) -> GroupKey { - group_key_from(&labels_with_name(metric), grouping) -} - -/// The group `labels` belong to, for callers that already hold the series' label pairs. -fn group_key_from(labels: &[(Arc, Arc)], grouping: &Grouping) -> GroupKey { - let mut key: Vec<_> = labels - .iter() + let mut key: Vec<_> = labels_with_name(metric) + .into_iter() .filter(|(name, value)| { !value.is_empty() && match grouping { @@ -666,7 +617,6 @@ fn group_key_from(labels: &[(Arc, Arc)], grouping: &Grouping) -> Group Grouping::Without(names) => &**name != "__name__" && !names.contains(name), } }) - .cloned() .collect(); key.sort(); key @@ -674,8 +624,6 @@ fn group_key_from(labels: &[(Arc, Arc)], grouping: &Grouping) -> Group /// The label pairs of `metric` with `__name__` among them, sorted by name. fn labels_with_name(metric: &SeriesLabels) -> Vec<(Arc, Arc)> { - #[cfg(test)] - calls::record_label_list(); let mut labels: Vec<_> = metric .tags() .iter() @@ -806,38 +754,3 @@ fn into_value(sample: SampleRef<'_>) -> Result> { fn cursor_value(cursor: &mut RowCursor<'_>, row: usize) -> Result> { into_value(cursor.at(row).expect("the axis and the fields are aligned")) } - -/// How often this thread built a series' label list or compared two candidates, so a test can hold -/// `topk` to the work it should not do rather than to the result, which every shape produces. -/// Test-only: a decorated sort and a partial selection select the same candidates. -#[cfg(test)] -pub(crate) mod calls { - use std::cell::Cell; - - thread_local! { - /// Label lists [`super::labels_with_name`] built on this thread. - static LABEL_LISTS: Cell = const { Cell::new(0) }; - /// Candidate pairs [`super::rank`] looked at on this thread. - static RANKS: Cell = const { Cell::new(0) }; - } - - /// Records one label list. - pub(crate) fn record_label_list() { - LABEL_LISTS.with(|lists| lists.set(lists.get() + 1)); - } - - /// Records one comparison of two candidates. - pub(crate) fn record_rank() { - RANKS.with(|ranks| ranks.set(ranks.get() + 1)); - } - - /// The label lists built on this thread since the last call, which starts a new count. - pub(crate) fn take_label_lists() -> usize { - LABEL_LISTS.with(|lists| lists.replace(0)) - } - - /// The candidate comparisons made on this thread since the last call, which starts a new count. - pub(crate) fn take_ranks() -> usize { - RANKS.with(|ranks| ranks.replace(0)) - } -} diff --git a/src/promql/src/select/tests.rs b/src/promql/src/select/tests.rs index 8e739a9563c..360d4d92445 100644 --- a/src/promql/src/select/tests.rs +++ b/src/promql/src/select/tests.rs @@ -24,7 +24,7 @@ use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis}; use datatypes::series::{Label, SeriesLabels}; use super::{ - Grouping, LabelMatcher, MatchOp, SortOrder, absent, absent_labels, calls, limit_ratio, limitk, + Grouping, LabelMatcher, MatchOp, SortOrder, absent, absent_labels, limit_ratio, limitk, natural_less, sample_offset, scalar, sort_instant, topk, }; use crate::golden::fold_pieces; @@ -33,7 +33,7 @@ use crate::grid_series::reference::{ run_ranges, sparse_series, }; use crate::grid_series::{GridSample, GridSeriesBuilder}; -use crate::histogram_eval::float_histogram::{FloatHistogram, copies}; +use crate::histogram_eval::float_histogram::FloatHistogram; fn schema(points: u64, fields: usize) -> SeriesSchema { SeriesSchema { @@ -498,148 +498,6 @@ fn limit_operators_keep_the_start_timestamp_of_a_selected_sample() { ); } -/// The selections that only move samples hand them from the field to the builder: no owned -/// histogram is built on the way, and the payload arrives as it was read. -#[test] -fn limit_operators_pass_histograms_through_without_an_owned_form() { - let histogram = - OwnedHistogram::parse("0/0/0/3ff0000000000000/3ff0000000000000/0/1_1/3ff0000000000000///"); - let schema = schema(3, 1); - let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone())).unwrap(); - builder - .push_refs(0, &[SampleRef::Histogram(histogram.view())]) - .unwrap(); - builder - .push_refs(2, &[SampleRef::Histogram(histogram.view())]) - .unwrap(); - let series = builder.finish(metric("__name__=a")).unwrap().unwrap(); - - copies::take_materialized(); - let by_ratio = limit_ratio(1.0, &schema, std::slice::from_ref(&series)).unwrap(); - let by_count = limitk( - 1.0, - &Grouping::By(Vec::new()), - &schema, - std::slice::from_ref(&series), - ) - .unwrap(); - assert_eq!( - copies::take_materialized(), - 0, - "a selection built an owned histogram of a sample it only moved" - ); - - // Both selections hand on the samples they kept, unchanged and in the same place. - let kept = |series: &Series| -> Vec<(usize, bool)> { - let samples = &series.fields.as_slice()[0]; - series - .time - .positions() - .map(|(point, row)| { - ( - point, - matches!(samples.sample(row), SampleRef::Histogram(_)), - ) - }) - .collect() - }; - for kept_series in [&by_ratio, &by_count] { - assert_eq!(kept_series.len(), 1); - assert_eq!(kept(&kept_series[0]), [(0, true), (2, true)]); - } -} - -/// `topk` builds one label list per series and ranks each point with a partial selection: the -/// comparisons it makes grow with the candidates rather than with their logarithm, which is what -/// ordering the whole group at every point cost. -#[test] -fn topk_ranks_a_point_with_a_partial_selection() { - let schema = schema(1, 1); - // Every series is one group, and the input arrives in reverse label order, so nothing about it - // is inherited from the order the members were built in. - let series: Vec = (0..256) - .rev() - .map(|index| { - let value = if index < 16 { 10.0 } else { 1.0 }; - build( - &schema, - metric(&format!("__name__=s{index:03}")), - &vec![vec![Some(value)]], - ) - .unwrap() - }) - .collect(); - - calls::take_label_lists(); - calls::take_ranks(); - let kept = topk(8.0, true, &Grouping::By(Vec::new()), &schema, &series).unwrap(); - let lists = calls::take_label_lists(); - let comparisons = calls::take_ranks(); - - let mut kept = rendered(&kept); - kept.sort(); - assert_eq!( - kept, - [ - "__name__=s000", - "__name__=s001", - "__name__=s002", - "__name__=s003", - "__name__=s004", - "__name__=s005", - "__name__=s006", - "__name__=s007" - ], - ); - assert_eq!( - lists, - series.len(), - "topk rebuilt a series' label list instead of reading the one it built" - ); - // Ordering the whole group at the point would take the candidates' logarithm per candidate; - // the partial selection takes a small constant. - assert!( - comparisons <= 4 * series.len(), - "topk compared {comparisons} candidate pairs to select 8 of {}", - series.len() - ); -} - -/// Ties go to the label set that sorts first, whatever order the input arrived in: the value the -/// partial selection stops at falls inside a run of equal values, and the run is resolved by label -/// order rather than by the order the ranking left the group in. -#[test] -fn topk_breaks_ties_by_label_order() { - let names: Vec = (0..40) - .map(|index| format!("__name__=s{index:02}")) - .collect(); - // The ten series that sort first carry the top value, and the input arrives in reverse label - // order, so a tie can only be resolved by the labels. - let values: Vec<(&str, Option)> = names - .iter() - .rev() - .map(|name| { - let value = if name.as_str() < "__name__=s10" { - 7.0 - } else { - 3.0 - }; - (name.as_str(), Some(value)) - }) - .collect(); - let (schema, series) = instant(&values); - let kept = topk(3.0, true, &Grouping::By(Vec::new()), &schema, &series).unwrap(); - let mut kept = rendered(&kept); - kept.sort(); - assert_eq!(kept, ["__name__=s00", "__name__=s01", "__name__=s02"]); - - // `bottomk` ties the same way, and its run of equal values holds the whole group. - let kept = topk(3.0, false, &Grouping::By(Vec::new()), &schema, &series).unwrap(); - let mut kept = rendered(&kept); - kept.sort(); - assert_eq!(kept, ["__name__=s10", "__name__=s11", "__name__=s12"]); -} - fn grouping(kind: &str, labels: &str) -> Grouping { match kind { "without" => Grouping::Without(names(labels)), diff --git a/src/promql/src/vector_match.rs b/src/promql/src/vector_match.rs index d31e64db155..f1c9e0db5a6 100644 --- a/src/promql/src/vector_match.rs +++ b/src/promql/src/vector_match.rs @@ -70,33 +70,6 @@ pub(crate) fn visited_pairs() -> usize { VISITED_PAIRS.with(std::cell::Cell::get) } -// How many matches the binary kernel recorded in this thread's match set. -// -// Test-only, and per thread: whether a field needs the set at all is what decides the per-sample -// work of the default one-to-one walk, and that is not visible in the result. A thread of its own -// keeps one test's matches out of another test's figure, and the counter is compiled out of the -// kernel otherwise. -#[cfg(test)] -thread_local! { - static MATCH_RECORDS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// Resets the match-set counter of this thread to zero. -#[cfg(test)] -pub(crate) fn reset_match_records() { - MATCH_RECORDS.with(|recorded| recorded.set(0)); -} - -/// The matches the binary kernel has recorded in this thread's match set since the last -/// [`reset_match_records`]. -/// -/// A match is recorded once per row the operator keeps, so the count is what the walk does rather -/// than what its operands' sizes imply it would. -#[cfg(test)] -pub(crate) fn match_records() -> usize { - MATCH_RECORDS.with(std::cell::Cell::get) -} - /// A PromQL binary operator between two instant vectors. #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Hash)] pub enum BinaryOp { @@ -895,21 +868,6 @@ fn binary_field( } let drop_name = !matching.op.is_comparison() || matching.return_bool; - // One-to-one: a match group may be matched once per point; Prometheus counts a filtered - // comparison as a match too, but not an operation that failed. Filling the one side consults the - // same set. A group whose many side holds one series is matched at a point by one pairing only -- - // a second row there would have to come from a second one-side series, and two members of one - // group present at one point is the duplicate `overlap` already reports -- so the set is built - // only for the fields that can hold two matches of one group at one point, and the walk of the - // default one-to-one shape records nothing at all. - let matched = fill.lhs.is_some() - || (matching.cardinality == Cardinality::OneToOne - && shared_match_group( - many, - &matching.labels, - metadata.tables.saturating_add(metadata.pairing), - budget.as_deref_mut(), - )?); let mut results = Results::default(); let mut pairs = Pairs { matching, @@ -917,7 +875,11 @@ fn binary_field( annotations, group_keys: Vec::new(), group_ids: HashMap::new(), - matched: matched.then(HashSet::new), + // One-to-one: a match group may be matched once per point; Prometheus counts a filtered + // comparison as a match too, but not an operation that failed. Filling the many side + // consults the same set. + matched: (matching.cardinality == Cardinality::OneToOne || fill.lhs.is_some()) + .then(HashSet::new), implicit_many_to_one: None, }; for (many_index, many_series) in many.series.iter().enumerate() { @@ -1147,96 +1109,6 @@ fn binary_field( Ok(results) } -/// Whether two series of `operand` share a match group. -/// -/// The keys are compared by a hash of the labels they keep: two series that share a group hash -/// equal, and two that differ may still collide -- an answer of `true` for a group that no two -/// series share only makes the caller build the set a shared group would need. One `u64` per series -/// is sorted in place and dropped with the check, so a field of single-member groups builds no key -/// and keeps none, whatever the length of the grid. -fn shared_match_group( - operand: &Operand<'_>, - labels: &MatchingLabels, - base: usize, - mut budget: Option<&mut Budget<'_>>, -) -> Result { - if operand.series.len() < 2 { - return Ok(false); - } - if let Some(budget) = budget.as_deref_mut() { - // One hash per series, entered beside the field's own tables before the list is allocated. - budget.phase( - base.saturating_add( - operand - .series - .len() - .saturating_mul(std::mem::size_of::()), - ), - )?; - } - let mut hashes: Vec = Vec::with_capacity(operand.series.len()); - hashes.extend( - operand - .series - .iter() - .map(|series| match_group_hash(&series.metric, labels)), - ); - hashes.sort_unstable(); - let shared = hashes.windows(2).any(|pair| pair[0] == pair[1]); - // The list is dropped with the check, so the field's own phase figure is live from here on. - if let Some(budget) = budget { - budget.phase(base)?; - } - Ok(shared) -} - -/// The FNV-1a offset basis and prime, which hash one label's name and value in the same steps -/// whatever hasher the tables of a field happen to use. -const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; -const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; - -/// The hash of the labels [`group_key`] would keep for `metric`. -/// -/// Each kept label hashes its name and value as one string, and the labels are summed with -/// wrapping addition, so the order the label set stores them in cannot change the answer: two -/// series of one match group hash equal however they are laid out. -fn match_group_hash(metric: &SeriesLabels, labels: &MatchingLabels) -> u64 { - let label = |name: &str, value: &str| { - let mut hash = FNV_OFFSET_BASIS; - for byte in name - .as_bytes() - .iter() - .chain(&[0xff]) - .chain(value.as_bytes()) - { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(FNV_PRIME); - } - hash - }; - let mut hash = 0u64; - match labels { - MatchingLabels::On(names) => { - for name in names { - // An empty value is absent, as in `group_key`. - if let Some(value) = metric.get(name) - && !value.is_empty() - { - hash = hash.wrapping_add(label(name, value)); - } - } - } - MatchingLabels::Ignoring(ignored) => { - for tag in metric.tags() { - if !tag.value.is_empty() && !ignored.contains(&tag.name) { - hash = hash.wrapping_add(label(&tag.name, &tag.value)); - } - } - } - } - hash -} - /// The matched pairs of one field and the state Prometheus checks them against. /// /// The results are filed beside it, not in it: a pairing's open row borrows the table it files into @@ -1248,10 +1120,8 @@ struct Pairs<'a> { group_keys: Vec, group_ids: HashMap, /// The match groups an operation matched at a point, kept where Prometheus consults them: - /// one-to-one matching, and filling the one side. A filtered comparison counts as a match, an - /// operation that failed does not. Kept only for the fields that need it: one whose groups hold - /// one many-side series cannot match a group twice at a point, and one filling neither side never - /// asks. + /// one-to-one matching, and filling the many side. A filtered comparison counts as a match, + /// an operation that failed does not. matched: Option>, implicit_many_to_one: Option<(usize, usize)>, } @@ -1325,11 +1195,6 @@ impl Pairs<'_> { // One-to-one: a match group may be matched once per point; Prometheus counts a filtered // comparison as a match too, but not an operation that failed. if let Some(matched) = &mut self.matched { - // Matches are what the set is for, and it exists only for the fields that can hold two - // of one group at one point: the test-only counter reports that the walk of a - // single-member group records nothing. - #[cfg(test)] - MATCH_RECORDS.with(|recorded| recorded.set(recorded.get() + 1)); // The match set is the one container this walk grows point by point, so it is charged by // the block it takes when it outgrows what it holds -- before the insert allocates it, and // never per point of a set that has room. diff --git a/src/promql/src/vector_match/tests.rs b/src/promql/src/vector_match/tests.rs index 62852393b4d..b0139ccb698 100644 --- a/src/promql/src/vector_match/tests.rs +++ b/src/promql/src/vector_match/tests.rs @@ -27,9 +27,8 @@ use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use super::{ - BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, RESULT_ROW_BYTES, SetOp, - binary_vectors, binary_vectors_with_reservation, match_records, reset_match_records, - set_vectors, set_vectors_with_reservation, + BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, SetOp, binary_vectors, + binary_vectors_with_reservation, set_vectors, set_vectors_with_reservation, }; use crate::grid_series::reference::{ Dense, OwnedHistogram, Points, assert_histogram_close, assert_result_eq, golden_cases, @@ -1200,163 +1199,6 @@ fn colliding_reset_hints_do_not_count_as_a_match() { } } -/// A group whose many side holds one series can be matched at a point by one pairing only: a second -/// row for that point would have to come from a second one-side series, and two members of one group -/// present at one point is the duplicate the one side's own scan reports. So the default one-to-one -/// shape records no match and builds no set, and the pairing still answers every point. -#[test] -fn a_single_series_group_records_no_match() { - const POINTS: usize = 1_000; - let grid = budget_grid(POINTS); - let schema = Arc::new(schema(1, &grid)); - let at: Vec = (0..POINTS).collect(); - // One series of the many side and one of the one side, in the same match group. - let lhs = vec![budget_operand(&schema, "a", &at)]; - let rhs = vec![budget_operand(&schema, "b", &at)]; - let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())); - reset_match_records(); - let output = - binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("the one pair matches"); - assert_eq!( - match_records(), - 0, - "a group of one many-side series recorded a match" - ); - // Both operands hold the point's own position, so the sum doubles it at every point. - assert_eq!(output.len(), 1, "{output:?}"); - let mut cursor = output[0].fields.as_slice()[0].cursor(); - let mut rows = 0; - for (point, row) in output[0].time.positions() { - match cursor.at(row).expect("the axis and the fields are aligned") { - SampleRef::Float(value) => assert_eq!(value, 2.0 * point as f64, "point {point}"), - _ => panic!("`+` of two floats keeps a float"), - } - rows += 1; - } - assert_eq!(rows, POINTS, "the pair answers every point"); -} - -/// A group of two many-side series can be matched twice at one point, so the set is built and the -/// second match of the group at that point is the duplicate Prometheus reports. -#[test] -fn a_group_of_two_many_series_records_its_matches() { - let grid = budget_grid(1); - let schema = Arc::new(schema(1, &grid)); - // Two left-hand series in one match group and one right-hand series they both pair with. - let lhs = [ - jobbed_operand(&schema, "a", "x", &[0]), - jobbed_operand(&schema, "b", "x", &[0]), - ]; - let rhs = [jobbed_operand(&schema, "c", "x", &[0])]; - let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())); - reset_match_records(); - let error = binary_vectors(&matching, &schema, &lhs, &schema, &rhs) - .unwrap_err() - .to_string(); - assert!( - error.contains("many-to-one matching must be explicit"), - "{error}" - ); - assert!( - match_records() >= 2, - "the second match of the group was not recorded: {}", - match_records() - ); -} - -/// Two one-side series of a group that are never present at the same point are no duplicate: each -/// point is matched by one pairing, which is what a group of one many-side series allows. -#[test] -fn a_group_matched_at_different_points_is_no_duplicate() { - const POINTS: usize = 64; - let grid = budget_grid(POINTS); - let schema = Arc::new(schema(1, &grid)); - let operand = |name: &str, instance: &str, value: f64, at: &[usize]| -> Series { - let labels = Labels::from([ - ("__name__".to_string(), name.to_string()), - ("job".to_string(), "x".to_string()), - ("instance".to_string(), instance.to_string()), - ]); - let points: Points = at - .iter() - .map(|&point| (point, f64::to_bits(value))) - .collect(); - sparse_series(&schema, &labels, &points).expect("the operand holds a value") - }; - let even: Vec = (0..POINTS).step_by(2).collect(); - let odd: Vec = (1..POINTS).step_by(2).collect(); - // One many-side series holding every point, and two one-side series that split the grid between - // them: the group is matched at every point, never twice at one. - let lhs = [operand("a", "many", 0.5, &(0..POINTS).collect::>())]; - let rhs = [ - operand("b", "even", 1.0, &even), - operand("c", "odd", 100.0, &odd), - ]; - let matching = budget_matching(BinaryOp::Add, MatchingLabels::On(vec![Arc::from("job")])); - reset_match_records(); - let output = - binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("no point is matched twice"); - assert_eq!( - match_records(), - 0, - "a group of one many-side series recorded a match" - ); - // One-to-one matching names the result after the many side and the matching labels, so both - // pairings file under one label set: the series holds the even points a `b` kept and the odd ones - // a `c` kept. - assert_eq!(output.len(), 1, "{output:?}"); - let mut cursor = output[0].fields.as_slice()[0].cursor(); - let mut rows = 0; - for (point, row) in output[0].time.positions() { - let expected = if point % 2 == 0 { 1.5 } else { 100.5 }; - match cursor.at(row).expect("the axis and the fields are aligned") { - SampleRef::Float(value) => assert_eq!(value, expected, "point {point}"), - _ => panic!("`+` of two floats keeps a float"), - } - rows += 1; - } - assert_eq!(rows, POINTS, "the two pairings cover the grid between them"); -} - -/// A fill value of the one side is answered from the matches the walk recorded, so a pairing that -/// holds at a point keeps it and only the points it left unmatched are filled -- the query of an -/// arbitrary point of a group, not a run of them. -#[test] -fn a_fill_consults_the_match_set_point_by_point() { - let grid = budget_grid(2); - let schema = Arc::new(schema(1, &grid)); - let lhs = [jobbed_operand(&schema, "a", "x", &[0])]; - let rhs = [jobbed_operand(&schema, "b", "x", &[0, 1])]; - let matching = BinaryMatching { - fill: FillValues { - lhs: Some(1_000.0), - rhs: None, - }, - ..budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())) - }; - reset_match_records(); - let output = - binary_vectors(&matching, &schema, &lhs, &schema, &rhs).expect("the operands are readable"); - // Filling the one side is what asks the set whether a point was matched, so the field builds it: - // the pair at point 0 records its match, and so does the row point 1 is filled with. - assert!( - match_records() > 0, - "filling the one side did not record the match it consults" - ); - assert_eq!(output.len(), 1, "{output:?}"); - let mut cursor = output[0].fields.as_slice()[0].cursor(); - let mut kept = Vec::new(); - for (point, row) in output[0].time.positions() { - match cursor.at(row).expect("the axis and the fields are aligned") { - // Point 0 is the pair's own sum; at point 1 the fill stands in for the sample the left - // side has at no point, and the operator is applied to it as it would be to a sample. - SampleRef::Float(value) => kept.push((point, value)), - _ => panic!("`+` of two floats keeps a float"), - } - } - assert_eq!(kept, [(0, 0.0), (1, 1_001.0)], "{output:?}"); -} - /// A pair whose operation fails is not a match, so `fill_left` still fills for the right-hand /// sample of that pair; the fixtures hold floats only. #[test] @@ -2615,43 +2457,6 @@ fn a_dense_result_is_refused_where_a_sparse_one_is_admitted() { ); } -/// A single field's rows reach the builder as they were filed: no table of triples is keyed by the -/// label set the rows already sit under, so what the merge adds to the rows the field holds is the -/// columns it writes them into, and a second copy of every sample is not part of the figure. -#[test] -fn a_single_field_holds_no_second_copy_of_its_rows() { - const POINTS: usize = 8_192; - let grid = budget_grid(POINTS); - let schema = Arc::new(schema(1, &grid)); - let at: Vec = (0..POINTS).collect(); - let lhs = vec![budget_operand(&schema, "a", &at)]; - let rhs = vec![budget_operand(&schema, "b", &at)]; - let matching = budget_matching(BinaryOp::Add, MatchingLabels::Ignoring(Vec::new())); - let (output, seen) = budgeted(&matching, &schema, &lhs, &schema, &rhs, usize::MAX); - let output = output.expect("the one pair matches"); - assert_eq!(output.len(), 1, "{output:?}"); - assert_eq!( - run_ranges(&output[0]).len(), - 1, - "the pair keeps one run of the whole grid" - ); - let peak = *seen - .iter() - .max() - .expect("the operation asks for its workspace"); - assert!(peak > 0, "the operation asks for its workspace"); - // What the operation holds once the field's tables are released is the rows it filed and the - // builder's own buffers: a merge that copied every row into a triple of its own table would hold - // a second copy of every sample beside them. - let floor = retained_total(&seen); - let rows = POINTS * RESULT_ROW_BYTES; - assert!(floor >= rows, "floor {floor}, rows {rows}"); - assert!( - floor <= rows + 8 * 1024, - "floor {floor} for {rows} row bytes: the merge holds a second copy of every sample" - ); -} - /// A field that meets its first histogram after a float backfills the index column of the rows /// before it, and the budget covers the column all the same. #[test] diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml index ea60ef78bd9..950f5da9e82 100644 --- a/src/query/Cargo.toml +++ b/src/query/Cargo.toml @@ -72,7 +72,6 @@ session.workspace = true snafu.workspace = true sql.workspace = true sqlparser.workspace = true -stacker.workspace = true store-api.workspace = true substrait.workspace = true table.workspace = true diff --git a/src/query/src/metrics.rs b/src/query/src/metrics.rs index 48f1937b62c..3c0043dcc09 100644 --- a/src/query/src/metrics.rs +++ b/src/query/src/metrics.rs @@ -126,36 +126,6 @@ lazy_static! { &["result"] ) .unwrap(); - - /// Statements the native PromQL route handed to the existing planner, by the cause it recorded. - /// - /// A declined statement is not an error and not a different answer: the existing planner - /// computes the same rows, so a fallback is only visible as a slower query. This counter and - /// the log line beside it are what make its rate observable per cause. - pub static ref NATIVE_PROMQL_DECLINED_TOTAL: IntCounterVec = register_int_counter_vec!( - "greptime_query_native_promql_declined_total", - "native PromQL expressions planned by the existing planner, by cause", - &["reason"] - ) - .unwrap(); - - /// Statements whose metadata drop the native PromQL planner collected for the scan's key - /// columns alone, by reason. - /// - /// The canonical pass collects the pieces of one label set before a consumer that reads whole - /// series, and it decides that from the labels a tree reports and from the key columns the scan - /// below them keys its series by. The second half is what one pair of stored series needs: a - /// primary-key column that holds `NULL` in one series and `''` in another decodes to an absent - /// and to an empty label, and a name-dropping kernel drops the empty one with the metadata - /// labels, leaving both under one label set. A drop the labels alone prove is collected for that - /// pair, and this counter is what makes the statements that pay for it observable. - pub static ref DUPLICATE_LABELSET_CHECK_SKIPPED_TOTAL: IntCounterVec = - register_int_counter_vec!( - "greptime_query_duplicate_labelset_check_skipped_total", - "native PromQL statements whose metadata drop was collected for their key columns alone, by reason", - &["reason"] - ) - .unwrap(); } /// A stream to call the callback once a RecordBatch stream is done. diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index c21b17e65f8..25c3a5352e8 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -22,10 +22,7 @@ pub use native::{NATIVE_PROMQL_EXTENSION_KEY, discovers_metric_name, native_prom mod set_operator; use std::collections::{BTreeSet, HashSet, VecDeque}; -use std::future::Future; -use std::pin::Pin; use std::sync::Arc; -use std::task::{Context, Poll}; use arrow::datatypes::IntervalDayTime; use async_recursion::async_recursion; @@ -220,45 +217,6 @@ struct PromPlannerContext { range_fold_offset: Option, } -/// Stack one planning descent level needs below it before the recursion grows its stack. -/// -/// The planner recurses once per operator and the descent is a chain of nested `poll`s on boxed -/// futures, so each level's frame is charged to the stack of the level above it. A debug build's -/// frame is large: a chain of eight constant multiplications under a propagated matcher needs more -/// than the 2 MiB a worker thread has, and there is no depth limit on the expressions this route -/// accepts. The recursion grows its stack instead, at the same reserve the native runtime's own -/// walks use. -const PLANNER_STACK_RED_ZONE: usize = 512 * 1024; - -/// Stack a growth adds when the reserve above is spent. It holds several levels of the descent. -const PLANNER_STACK_GROWTH: usize = 4 * 1024 * 1024; - -/// Polls one planning descent on a stack that grows on demand. -/// -/// `stacker::maybe_grow` needs a synchronous callee to run on the segment it switches to, which an -/// `async` body cannot be, so the growth wraps a `poll` instead: the segment stays live for that -/// whole poll, every nested level the poll reaches included, and each nested level wraps its own -/// poll in turn. This is the same reserve-and-grow rule `promql::native::execute` applies to the -/// runtime node walks, which recurse synchronously. -struct GrownStack(Pin>); - -impl GrownStack { - fn new(future: F) -> Self { - Self(Box::pin(future)) - } -} - -impl Future for GrownStack { - type Output = F::Output; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut inner = self.get_mut().0.as_mut(); - stacker::maybe_grow(PLANNER_STACK_RED_ZONE, PLANNER_STACK_GROWTH, || { - inner.as_mut().poll(cx) - }) - } -} - /// Result labels a vector-vector binary operation derives from its matching modifier, projected /// from the operand each label belongs to. #[derive(Debug)] @@ -346,14 +304,6 @@ pub struct PromPlanner { /// The engine-wide `QueryOptions::experimental_native_promql` is applied while `QueryEngineState` /// is built; this planner reads the per-query request extension instead. native_fused: bool, - /// Whether the binary node being planned may still offer its own subtree to the native route. - /// - /// An offer compiles a whole tree, so it is made where it covers the most: the statement root, - /// and the topmost binary below a node this route does not answer. Once a binary node has spent - /// the offer, the operands below it are planned by the existing path without offering - /// themselves again - re-offering each operand of a refused chain of n operators compiles its - /// subtrees again and costs O(n^2) passes instead of O(n). - native_binary_offer: bool, } type BinaryFieldPair<'a> = (&'a String, &'a String); @@ -386,9 +336,6 @@ impl PromPlanner { ctx: PromPlannerContext::from_eval_stmt(stmt)?, promql_annotations, native_fused, - // The statement's own expression is offered below before anything else is planned, so - // the first binary node reached still holds the offer a wrapper would have spent. - native_binary_offer: true, }; // The whole statement is offered to the native compiler first, because that compiler folds @@ -433,32 +380,6 @@ impl PromPlanner { prom_expr: &PromExpr, timestamp_fn: bool, query_engine_state: &QueryEngineState, - ) -> Result { - // Every operator of the existing expression planning descends through here, so this is - // where that recursion grows its stack instead of spending the caller's, below the reserve - // [`PLANNER_STACK_RED_ZONE`] names. Two limits come with it: - // - // - The native fused-series route is no part of this recursion. Its own composition - // recursion grows at its own boundary (`PromPlanner::try_plan_native` in `native.rs`), - // not here. - // - The synchronous walks around it are not covered at all: the literal construction - // (`try_build_literal_expr`) and the logical row-key walk - // (`collect_row_key_tag_columns_from_plan`) in this file, and the topology walk - // (`native_raw_plan_bounds_region_reads`), the root-parenthesis traversal - // (`root_subquery`), the input-order dependency walk (`depends_on_input_order`), the - // selector collection (`collect_selectors_with_set_operations`) and the native root - // subquery compilation in `native.rs`, recurse synchronously on the stack their caller - // has. A deep enough expression or plan still costs them that stack. - let step = self.prom_expr_to_plan_step(prom_expr, timestamp_fn, query_engine_state); - GrownStack::new(step).await - } - - #[async_recursion] - async fn prom_expr_to_plan_step( - &mut self, - prom_expr: &PromExpr, - timestamp_fn: bool, - query_engine_state: &QueryEngineState, ) -> Result { // An anchored range call is step-invariant: evaluate it once, at the start of the // evaluation, and report its result at every step; see @@ -480,18 +401,8 @@ impl PromPlanner { .await? } PromExpr::Binary(expr) => { - // One native offer per binary subtree: the offer at a node covers every node below - // it, because the compiler reads the whole tree at once. It is spent here, by the - // node that makes it, and handed back once the subtree is planned, so a refused - // chain of n operators makes one offer per position instead of one per operator - // and per subtree below it - O(n) instead of O(n^2). A wrapper this route cannot - // answer leaves the offer to the binary that sits under it. - let offer = std::mem::take(&mut self.native_binary_offer); - let plan = self - .prom_binary_expr_to_plan(query_engine_state, expr, offer) - .await; - self.native_binary_offer = offer; - plan? + self.prom_binary_expr_to_plan(query_engine_state, expr) + .await? } PromExpr::Paren(ParenExpr { expr }) => { self.prom_expr_to_plan_inner(expr, timestamp_fn, query_engine_state) @@ -1275,13 +1186,10 @@ impl PromPlanner { }) } - /// Plans `binary_expr`; `offer` says whether this node may still offer its subtree to the - /// native route, which the caller spends for the subtree it is about to plan. async fn prom_binary_expr_to_plan( &mut self, query_engine_state: &QueryEngineState, binary_expr: &PromBinaryExpr, - offer: bool, ) -> Result { // promql-parser accepts fill modifiers, but Greptime does not implement the // required outer joins and missing-value substitution. Reject them before the @@ -1298,7 +1206,6 @@ impl PromPlanner { // The same whole native binary the statement root was offered can sit below a // legacy-only wrapper. A nested retry is valid only on the statement's own grid. if self.native_fused - && offer && self.ctx.start == self.ctx.stmt_start && self.ctx.end == self.ctx.stmt_end { @@ -1553,32 +1460,19 @@ impl PromPlanner { // A copied matcher belongs to the scan, not to the operand's identity: // `absent()` turns `selector_matcher` into the labels it reports, so the // re-planned context keeps the matchers the operand was written with. - // - // A rewritten operand is an expression this node's offer never compiled: that - // offer read the statement as it was written, and the matcher reached the scan - // afterwards. The re-plan is therefore handed the offer, which the rewritten - // operand then spends on its own subtree exactly as this node spent it - so a - // chain below the rewrite still costs one offer, not one per operator and not - // one per operator and subtree below it. if rewritten.lhs.as_ref() != lhs.as_ref() { let selectors = std::mem::take(&mut left_context.selector_matcher); - let offer = std::mem::replace(&mut self.native_binary_offer, true); - let replanned = self + left_input = self .prom_expr_to_plan(&rewritten.lhs, query_engine_state) - .await; - self.native_binary_offer = offer; - left_input = replanned?; + .await?; left_context = self.ctx.clone(); left_context.selector_matcher = selectors; } if rewritten.rhs.as_ref() != rhs.as_ref() { let selectors = std::mem::take(&mut right_context.selector_matcher); - let offer = std::mem::replace(&mut self.native_binary_offer, true); - let replanned = self + right_input = self .prom_expr_to_plan(&rewritten.rhs, query_engine_state) - .await; - self.native_binary_offer = offer; - right_input = replanned?; + .await?; right_context = self.ctx.clone(); right_context.selector_matcher = selectors; } diff --git a/src/query/src/promql/planner/native.rs b/src/query/src/promql/planner/native.rs index a0bec6ff492..ad0043bf5d2 100644 --- a/src/query/src/promql/planner/native.rs +++ b/src/query/src/promql/planner/native.rs @@ -39,7 +39,6 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE, MITO2_ENGINE}; use common_query::native_histogram::native_histogram_value_type; use common_query::prelude::OTLP_AGGREGATION_TEMPORALITY_LABEL; use common_query::series_frame::raw_series_frame_schema_with_ordering; -use common_telemetry::debug; use datafusion::arrow::datatypes::DataType as ArrowDataType; use datafusion::common::{DFSchema, DFSchemaRef}; use datafusion::error::DataFusionError; @@ -55,7 +54,7 @@ use datatypes::series::time::EvalGrid; use futures::future::BoxFuture; use promql::extension_plan::{ FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec, InputLocality, - RawSeriesSource, input_locality, keys_survive_metadata_drop, matching_operands_drop_key, + RawSeriesSource, input_locality, matching_operands_drop_key, }; use promql::native::plan::{ AggregateKind, BoundAggregate, BoundInput, BoundVectorOp, NativeExpr, NativeNode, NativeTree, @@ -65,7 +64,7 @@ use promql::native::plan::{ use promql::native::segment::{ self, NATIVE_ORDERING, canonical_boundary_stages, canonical_local_prefix, contains_canonical_merge, contains_global_consumer, groups_by_metric_name, - region_local_segment, segment_tree, with_row_name_column, + merge_answers_histogram, region_local_segment, segment_tree, with_row_name_column, }; // Kept reachable here because this crate reads them through this module's own path: the dist // planner's commutativity check asks whether a fused node is per-series work, and this crate's @@ -88,9 +87,7 @@ use crate::promql::native_compile::{ InfoInput, bind_evaluated_binary, compile_selectors_with_grid, compile_statement_with_grid, info_input, statement_grid, subquery_child_grid, }; -use crate::promql::planner::{ - DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, GrownStack, PromPlanner, -}; +use crate::promql::planner::{DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, PromPlanner}; use crate::query_engine::QueryEngineState; /// The query option that turns the native fused-series route on for one query. @@ -186,13 +183,6 @@ type PreparedInput = ( // `split_series`: whether that partition splits one series across regions, which a bare scan // cannot reconcile. A partition on tags keeps every series whole in one region. bool, - // `keys_survive`: whether the resolved table keys its series by columns that always decode a - // value, read from its own primary key through `keys_survive_metadata_drop`. It is the half of - // the metadata drop's proof a tree cannot answer about itself: the drop removes every empty - // label value with the name, so a column holding `NULL` in one series and `''` in another - // leaves both under one label set. The canonical pass collects the drop when this is `false`, - // and `SKIPPED_LABELSET_CHECK_NULLABLE_KEY` counts the statements collected for that reason. - bool, ); /// A fully placed native operand and the evaluated layout its frame carries. @@ -230,86 +220,6 @@ fn native_planned( } } -thread_local! { - /// The cause the native route recorded for the expression it is about to refuse. - /// - /// A refusal is answered by four different walks - the compiler binding, the scan preparation, - /// the output-schema derivation and the segmentation - and most of those sites are plain - /// functions that can only answer `None`. The cause therefore travels beside the `None` rather - /// than through it: a refusal records itself here, and the offer that receives the `None` - /// reports it. One statement is planned on one thread, and taking the record clears it. - static NATIVE_DECLINE: std::cell::Cell> = - const { std::cell::Cell::new(None) }; - /// How many native attempts this thread made, one per whole-expression offer. - /// - /// An attempt compiles the whole tree below it, so this count times the statement's size bounds - /// the work planning it costs. A test uses it to hold a refused chain to one attempt per - /// position instead of one per position and per operator below it, which is O(n^2). - #[cfg(test)] - static NATIVE_ATTEMPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; - /// The cause the route last reported for a refused offer. - /// - /// The report is a log line and a counter, neither of which a test can read back; this holds - /// the same cause the two of them carry. - #[cfg(test)] - static REPORTED_DECLINE: std::cell::Cell> = - const { std::cell::Cell::new(None) }; - /// The reason this thread's statement had its metadata drop collected for its key columns. - /// - /// The canonical pass decides that while it rewrites a tree, and the counter that reports it is - /// taken once per statement by the offer that answers the plan - so the reason travels beside - /// the tree the way a refusal travels beside the `None` it answers. - static SKIPPED_LABELSET_CHECK: std::cell::Cell> = - const { std::cell::Cell::new(None) }; - /// The reason the counter last reported for a drop collected over a nullable key. - /// - /// The counter is not something a test can read back; this holds the same reason it carries. - #[cfg(test)] - static REPORTED_SKIPPED_LABELSET_CHECK: std::cell::Cell> = - const { std::cell::Cell::new(None) }; -} - -/// The cause reported when a refusal was answered without recording one. -const NATIVE_DECLINE_UNRECORDED: &str = "cause_not_recorded"; - -/// The reason a statement's metadata drop was collected: the scan that answers it keys its series by -/// a column that can hold `NULL` in one series and `''` in another, so the labels alone cannot prove -/// the drop tells two of its series apart. -const SKIPPED_LABELSET_CHECK_NULLABLE_KEY: &str = "nullable_key"; - -/// Answers the `None` that hands this expression to the existing planner, recording why. -/// -/// Declining is the normal answer to a shape this route does not answer: the existing planner -/// computes the same rows, so the site itself stays quiet. What it contributes is the cause, which -/// the offer that receives the `None` reports once. -fn declined(reason: &'static str) -> Option { - NATIVE_DECLINE.with(|record| record.set(Some(reason))); - None -} - -/// Takes the cause the last refusal recorded, if it recorded one. -fn take_native_decline() -> Option<&'static str> { - NATIVE_DECLINE.with(|record| record.replace(None)) -} - -/// Clears the attempt count and answers how many attempts this thread made since the last call. -#[cfg(test)] -pub(crate) fn take_native_attempts() -> usize { - NATIVE_ATTEMPTS.with(|attempts| attempts.replace(0)) -} - -/// Takes the cause the route last reported for a refused offer. -#[cfg(test)] -pub(crate) fn take_reported_decline() -> Option<&'static str> { - REPORTED_DECLINE.with(|reported| reported.replace(None)) -} - -/// Takes the reason the route last reported for a drop collected over a nullable key. -#[cfg(test)] -pub(crate) fn take_reported_skipped_labelset_check() -> Option<&'static str> { - REPORTED_SKIPPED_LABELSET_CHECK.with(|reported| reported.replace(None)) -} - impl PromPlanner { /// Builds the scan of the one logical table the current context is bound to. The caller has /// already preprocessed this selector's matchers and set the context to its table. @@ -326,14 +236,14 @@ impl PromPlanner { self.ctx.table_name.clone(), self.ctx.time_index_column.clone(), ) else { - return Ok(declined("selector_without_table_or_time_index")); + return Ok(None); }; let tags = self.ctx.tag_columns.clone(); let fields = self.ctx.field_columns.clone(); // A `__field__` matcher narrows the field set in a projection the existing planner builds, and the // raw series scan below this node has no equivalent: it reports every field the scan carries. if self.ctx.field_column_matcher.is_some() || fields.is_empty() { - return Ok(declined("field_selector_without_raw_series_equivalent")); + return Ok(None); } // Delta samples are increments: the existing planner sums them for `increase` and // divides by the range for `rate`, while the native tree assumes cumulative samples. @@ -343,7 +253,7 @@ impl PromPlanner { .any(|tag| tag == OTLP_AGGREGATION_TEMPORALITY_LABEL) && under_rate_or_increase { - return Ok(declined("delta_temporality_under_rate_or_increase")); + return Ok(None); } // The selector's own table scan, built by the same method the existing path uses: // the physical planner turns it into the region scan the node reads, and building a @@ -372,7 +282,7 @@ impl PromPlanner { .map(|field| source_schema.index_of(field).ok()) .collect::>>() else { - return Ok(declined("field_missing_from_source_schema")); + return Ok(None); }; let original_projection = scan.projection.as_ref(); let selected_indices = original_projection @@ -396,7 +306,7 @@ impl PromPlanner { } None => match &input { LogicalPlan::TableScan(scan) => scan.clone(), - _ => return Ok(declined("selector_plan_is_not_a_table_scan")), + _ => return Ok(None), }, }; let table = self.table_from_source(&scan.source)?; @@ -405,18 +315,6 @@ impl PromPlanner { // only collects the pieces its own producer holds, so the caller needs this fact to tell a whole-query // label-set answer from one region's share of it. let partitioned = table_meta.partition_column_names().next().is_some(); - // A kernel that drops the metadata labels drops every label whose value is empty with them, - // so two stored series whose key columns hold `NULL` and `''` leave it as one identity. This - // is the fact the drop's proof needs beside the labels - the same rule the physical - // per-series scan reads to decide whether a node may stream - and the canonical pass - // collects the drop when it does not hold. - let keys_survive = keys_survive_metadata_drop( - tags.iter().map(String::as_str), - table_meta - .primary_key_indices - .iter() - .map(|index| &table_meta.schema.column_schemas()[*index]), - ); // An engine without a raw-series scanner has no frame to give this node, and by the time the physical // planner asks, the logical plan is already this node: the refusal there is a planning error, not a // fallback. So the engine is checked here. A file-engine table is the concrete case, since its @@ -425,7 +323,7 @@ impl PromPlanner { if !supports_native_raw_series(&table_meta.engine) || (table_meta.engine == METRIC_ENGINE_NAME && !has_verified_metric_rewrite) { - return Ok(declined("engine_without_raw_series_scanner")); + return Ok(None); } // The region's raw-series preflight refuses these, and the physical layer turns that // refusal into an error, so decline this route here instead. @@ -441,7 +339,7 @@ impl PromPlanner { .get(store_api::mito_engine_options::APPEND_MODE_KEY) .is_some_and(|value| value.eq_ignore_ascii_case("true")) { - return Ok(declined("unsupported_time_index_type_or_append_mode")); + return Ok(None); } // A partition column that is not a tag splits one series across regions, and a bare scan cannot // reconcile the fragments; a raw source can, so such a selector is admitted and its input is bound per @@ -462,10 +360,10 @@ impl PromPlanner { .iter() .map(|field| Self::field_column_type(input.schema(), field)), ) else { - return Ok(declined("mixed_field_kinds")); + return Ok(None); }; if statement_kind.is_some_and(|kind| kind != field_kind) { - return Ok(declined("selectors_disagree_on_field_kind")); + return Ok(None); } *statement_kind = Some(field_kind); let series_schema = Arc::new(SeriesSchema { @@ -485,7 +383,7 @@ impl PromPlanner { if let LogicalPlan::TableScan(table_scan) = &mut input { table_scan.filters.append(&mut filters); } else { - return Ok(declined("selector_plan_is_not_a_table_scan")); + return Ok(None); } Ok(Some(( input, @@ -495,7 +393,6 @@ impl PromPlanner { tags, partitioned, split_series, - keys_survive, ))) } @@ -504,48 +401,14 @@ impl PromPlanner { prom_expr: &PromExpr, query_engine_state: &QueryEngineState, ) -> Result> { - // With the route off nothing below runs, so reporting the answer as a fallback would - // describe a route the query never asked for. - if !self.native_fused { - return Ok(None); - } - let planned = self + match self .try_plan_native(prom_expr, query_engine_state, FusedOutputFormat::PlainRows) - .await?; - // Whether this statement is one whose metadata drop the canonical pass collected for its - // key columns alone. The attempts below record it where they decide it; taking it here - // keeps the count one per statement and leaves nothing behind for the next one. - let skipped = SKIPPED_LABELSET_CHECK.with(|skipped| skipped.replace(None)); - match planned { + .await? + { // A whole statement answers a row table. A frame is what a composition reads from its // operands, so the statement root is never one. - Some(NativePlanned::Rows(plan)) => { - if let Some(reason) = skipped { - crate::metrics::DUPLICATE_LABELSET_CHECK_SKIPPED_TOTAL - .with_label_values(&[reason]) - .inc(); - #[cfg(test)] - REPORTED_SKIPPED_LABELSET_CHECK.with(|reported| reported.set(Some(reason))); - } - Ok(Some(plan)) - } - _ => { - // A declined expression is planned by the existing path and answers the same rows, - // so without this record a route that never ran would only be visible as a slower - // query. The cause is the refusal that decided the answer; a refusal from a walk - // that records nothing reports the generic cause rather than staying silent. - let reason = take_native_decline().unwrap_or(NATIVE_DECLINE_UNRECORDED); - debug!( - "Native PromQL route declined this expression ({reason}); \ - the existing planner answers it" - ); - crate::metrics::NATIVE_PROMQL_DECLINED_TOTAL - .with_label_values(&[reason]) - .inc(); - #[cfg(test)] - REPORTED_DECLINE.with(|reported| reported.set(Some(reason))); - Ok(None) - } + Some(NativePlanned::Rows(plan)) => Ok(Some(plan)), + _ => Ok(None), } } @@ -560,26 +423,13 @@ impl PromPlanner { /// /// `Box::pin` is here only because a composition and this attempt call each other, which an anonymous /// async future cannot be sized for. - /// - /// The boxed future is polled through [`GrownStack`], which is the one place this route's recursion - /// grows its stack: a composition offers each operand here, so the descent alternates between this - /// attempt and [`Self::try_plan_native_composed_binary`], and every one of those levels is charged to - /// the stack of the level above it, below the reserve `PLANNER_STACK_RED_ZONE` names. fn try_plan_native<'a>( &'a mut self, prom_expr: &'a PromExpr, query_engine_state: &'a QueryEngineState, format: FusedOutputFormat, ) -> BoxFuture<'a, Result>> { - // One attempt is one whole-expression offer: a test counts them to hold the cost of a - // refused chain to its own positions, and the report below counts what a query paid for. - #[cfg(test)] - NATIVE_ATTEMPTS.with(|attempts| attempts.set(attempts.get() + 1)); - // An attempt descends the tree it was offered once per operator, and the composition below - // offers each operand in turn, so the recursion grows its stack here as well as at the - // planner's own descent: `GrownStack` grows it for the whole poll of this future, which - // covers every nested attempt the poll reaches. - Box::pin(GrownStack::new(async move { + Box::pin(async move { let saved = self.ctx.clone(); let whole = Box::pin(self.try_plan_native_fused_with_context( prom_expr, @@ -598,7 +448,7 @@ impl PromPlanner { .await; self.ctx = saved; composed - })) + }) } /// Combines two native operands without a legacy join. @@ -615,9 +465,6 @@ impl PromPlanner { query_engine_state: &QueryEngineState, format: FusedOutputFormat, ) -> Result> { - // Neither check refuses a shape: they say this second attempt does not apply here, so they - // record no cause - the whole-expression attempt above already named the one that decided - // the answer. if !self.native_fused || self.ctx.start != self.ctx.stmt_start || self.ctx.end != self.ctx.stmt_end @@ -645,7 +492,7 @@ impl PromPlanner { .await; self.ctx = saved.clone(); let Some(NativePlanned::Frame(left)) = left? else { - return Ok(declined("left_operand_is_not_a_frame")); + return Ok(None); }; let right = self .try_plan_native( @@ -656,10 +503,10 @@ impl PromPlanner { .await; self.ctx = saved; let Some(NativePlanned::Frame(right)) = right? else { - return Ok(declined("right_operand_is_not_a_frame")); + return Ok(None); }; if left.grid.spec() != right.grid.spec() { - return Ok(declined("operand_grids_differ")); + return Ok(None); } let NativeFramePlan { plan: left_plan, @@ -691,13 +538,11 @@ impl PromPlanner { evaluated(1, &right_output), ) { Ok(tree) => tree, - Err(DataFusionError::NotImplemented(_)) => { - return Ok(declined("no_kernel_for_the_operator_pair")); - } + Err(DataFusionError::NotImplemented(_)) => return Ok(None), Err(error) => return Err(error).context(DataFusionPlanningSnafu), }; let NativeExpr::Binary { matching, .. } = &tree.root().expr else { - return Ok(declined("bound_root_is_not_the_binary")); + return Ok(None); }; let Some(root_kind) = binary_output_field_kind( left_kind, @@ -706,10 +551,10 @@ impl PromPlanner { matching.return_bool, matching.fill, ) else { - return Ok(declined("no_output_kind_for_the_pair")); + return Ok(None); }; if !native_tree_supported(&tree) { - return Ok(declined("runtime_refuses_the_composed_tree")); + return Ok(None); } // The expression names the value columns, and it is the label the node reports. let expression = expr.to_string(); @@ -720,7 +565,7 @@ impl PromPlanner { FusedOutputFormat::PlainRows => None, FusedOutputFormat::SeriesFrame => { let Some(tags) = fused_output_tags(tree.root(), &source_tags) else { - return Ok(declined("composition_has_no_output_labels")); + return Ok(None); }; Some((tree.root().output.clone(), tags)) } @@ -735,7 +580,7 @@ impl PromPlanner { root_kind, )? else { - return Ok(declined("composition_has_no_value_columns")); + return Ok(None); }; schema } @@ -743,9 +588,7 @@ impl PromPlanner { let frame = match FusedSeries::series_frame_schema(&tree) { Ok(frame) => frame, // A frame this composition cannot declare is a shape it does not answer. - Err(DataFusionError::Plan(_)) => { - return Ok(declined("composition_cannot_declare_a_frame")); - } + Err(DataFusionError::Plan(_)) => return Ok(None), Err(error) => return Err(error).context(DataFusionPlanningSnafu), }; Arc::new( @@ -766,7 +609,7 @@ impl PromPlanner { output_schema, format, ) else { - return Ok(declined("fused_node_refuses_the_composition")); + return Ok(None); }; let plan = LogicalPlan::Extension(Extension { node: Arc::new(node), @@ -796,7 +639,7 @@ impl PromPlanner { format: FusedOutputFormat, ) -> Result> { if !self.native_fused { - return Ok(declined("native_route_not_requested")); + return Ok(None); } // The expression names the value columns of every rows layout this attempt publishes, and the // node's own label follows the same text. It is the whole statement's expression even when a @@ -809,23 +652,23 @@ impl PromPlanner { if prom_expr.value_type() == ValueType::Matrix && (subquery_root.is_none() || self.ctx.start != self.ctx.end) { - return Ok(declined("matrix_root_without_a_grid")); + return Ok(None); } let Some(stmt) = self.native_statement(prom_expr) else { - return Ok(declined("statement_has_no_evaluation_grid")); + return Ok(None); }; // The subquery root's inner expression is compiled on the child grid, with the statement left // untouched: `@ start()` and `@ end()` inside it resolve against the statement's bounds. A statement // whose grid cannot be built keeps the existing plan. let child_grid = if let Some(subquery) = subquery_root { let Ok(outer) = statement_grid(&stmt) else { - return Ok(declined("subquery_outer_grid_unavailable")); + return Ok(None); }; let Some(child) = subquery_child_grid(&stmt, &outer, subquery).context(DataFusionPlanningSnafu)? else { - return Ok(declined("subquery_has_no_child_grid")); + return Ok(None); }; Some(child.grid) } else { @@ -847,12 +690,12 @@ impl PromPlanner { &mut selectors, &mut set_operations, ) { - return Ok(declined("selector_walk_refuses_the_statement")); + return Ok(None); } // The compiler decides which calls and shapes this route answers: a name it has no kernel for comes // back as `NotImplemented`, which is a decline below. let Ok(statement) = statement_grid(&stmt) else { - return Ok(declined("statement_grid_unavailable")); + return Ok(None); }; // The one grid every lowering path below binds: a subquery root is compiled on its child // grid, and the rows it publishes carry that grid, so the split, the boundary and the @@ -863,9 +706,7 @@ impl PromPlanner { compile_stmt.expr = effective_expr.clone(); let tree = match compile_statement_with_grid(&compile_stmt, Arc::clone(&grid)) { Ok(tree) => tree, - Err(DataFusionError::NotImplemented(_)) => { - return Ok(declined("no_kernel_for_the_statement_shape")); - } + Err(DataFusionError::NotImplemented(_)) => return Ok(None), Err(error) => return Err(error).context(DataFusionPlanningSnafu), }; // A frame operand is an evaluated sample root: a composition matches series, and a @@ -876,16 +717,16 @@ impl PromPlanner { NativeType::Samples(schema) if schema.eval_grid.is_some() ); if format == FusedOutputFormat::SeriesFrame && !evaluated { - return Ok(declined("frame_operand_is_not_evaluated")); + return Ok(None); } // Bind only input tags: this data-free tree has no input, and its tags belong to its // output schema. let Some(output_tags) = fused_output_tags(tree.root(), &[]) else { - return Ok(declined("data_free_root_has_no_labels")); + return Ok(None); }; let Some(values) = native_root_fields(tree.root()) else { - return Ok(declined("data_free_root_has_no_value_columns")); + return Ok(None); }; // A label named `time` is the one collision the default time column can have: the // value columns are named after the expression, so they never take that name. The @@ -976,7 +817,7 @@ impl PromPlanner { // These matchers are alternatives the one scan below the node cannot represent, so the // query keeps the plan the existing path builds for it. if !selector.matchers.or_matchers.is_empty() { - return Ok(declined("or_matchers_have_no_single_raw_scan")); + return Ok(None); } // A selector that names no metric is answered by enumerating the schema's logical metric // tables: one input per metric, each reading its own region, so every series keeps the @@ -994,7 +835,7 @@ impl PromPlanner { .preprocess_label_matchers(&ordinary_matchers, &Some(String::new())) { Ok(m) => m, - Err(_) => return Ok(declined("discovery_matchers_cannot_be_preprocessed")), + Err(_) => return Ok(None), }; let effective_schema = self .ctx @@ -1005,7 +846,7 @@ impl PromPlanner { // see never becomes a source, so its name reaches neither the plan nor an error. let scope = self.table_provider.query_ctx().metric_discovery_scope(); if matches!(scope, MetricDiscoveryScope::Unavailable) { - return Ok(declined("metric_discovery_unavailable")); + return Ok(None); } let mut metrics = self .table_provider @@ -1071,7 +912,7 @@ impl PromPlanner { // is declined for a scan this route can read, and the same refusal keeps the empty // input from claiming a field layout the matcher would have decided. if self.ctx.field_column_matcher.is_some() { - return Ok(declined("field_selector_on_an_empty_input")); + return Ok(None); } prepared.push(empty_discovered_input( self.ctx @@ -1089,14 +930,10 @@ impl PromPlanner { // makes the whole input partitioned. let mut discovered_partitioned = false; let mut discovered_split_series = false; - // Every discovered metric's own key columns decide its series, so the union - // survives the drop only when all of them do - and a metric whose keys do not - // is enough to collect the whole input. - let mut discovered_keys_survive = true; for table in metrics { self.ctx.table_name = Some(table.table_info().name.clone()); if self.setup_context().await?.is_some() { - return Ok(declined("discovered_table_has_no_data")); + return Ok(None); } let Some(( scan, @@ -1106,7 +943,6 @@ impl PromPlanner { mut tags, partitioned, split_series, - keys_survive, )) = self .prepare_one_scan( matchers.clone(), @@ -1117,17 +953,14 @@ impl PromPlanner { ) .await? else { - // A refusal here is the selector's own: `prepare_one_scan` recorded - // its cause, and naming this pass again would overwrite it. return Ok(None); }; discovered_partitioned |= partitioned; discovered_split_series |= split_series; - discovered_keys_survive &= keys_survive; // The union reads every discovered metric, so it is region-local only when // every metric it reads is. let Some(metric) = metric else { - return Ok(declined("discovered_selector_names_no_metric")); + return Ok(None); }; // The expression names no metric, so each source reports its own: the tag is // what tells two metrics with equal ordinary labels apart, and it is what a @@ -1180,7 +1013,7 @@ impl PromPlanner { } } let Some((schema, time, mut tags, single_metric)) = selector_metadata else { - return Ok(declined("discovered_selectors_disagree_on_their_layout")); + return Ok(None); }; tags.sort(); tags.dedup(); @@ -1205,17 +1038,16 @@ impl PromPlanner { tags, discovered_partitioned, discovered_split_series, - discovered_keys_survive, )); continue; } let matchers = match self.preprocess_label_matchers(&selector.matchers, &selector.name) { Ok(m) => m, - Err(_) => return Ok(declined("selector_matchers_cannot_be_preprocessed")), + Err(_) => return Ok(None), }; if self.setup_context().await?.is_some() { - return Ok(declined("selector_has_no_data")); + return Ok(None); } let Some(one) = self .prepare_one_scan( @@ -1227,7 +1059,6 @@ impl PromPlanner { ) .await? else { - // `prepare_one_scan` recorded why, above. return Ok(None); }; prepared.push(one); @@ -1237,7 +1068,6 @@ impl PromPlanner { .await; self.ctx = original_context; let Some((mut prepared, logical_ids_differ, statement_kind)) = preparation? else { - // The selector that refused recorded its own cause; this pass only carries it up. return Ok(None); }; // `absent_over_time` reads its window through a fold that keeps the metric name, so the segment that @@ -1252,7 +1082,7 @@ impl PromPlanner { .iter() .any(|(scan, _, metric, ..)| !is_empty_discovered_input(scan) && metric.is_none()) { - return Ok(declined("presence_over_a_nameless_selector")); + return Ok(None); } // Selectors over one table share one region, so the node runs there over every scan. When they read // more than one table, each scan answers from its own region and the node runs above them; see @@ -1297,12 +1127,12 @@ impl PromPlanner { prepared[index].1 = Arc::clone(&schema); } // A real input names the output's time index and, on one table, binds every scan. - let Some((_, _, metric, time, tags, _, _, _)) = prepared + let Some((_, _, metric, time, tags, _, _)) = prepared .iter() .find(|(scan, ..)| !is_empty_discovered_input(scan)) .or_else(|| prepared.first()) else { - return Ok(declined("statement_prepared_no_readable_input")); + return Ok(None); }; let (metric, time, tags) = (metric.clone(), time.clone(), tags.clone()); // The compiler binds one input per selector in the same pre-order `collect_selectors` @@ -1367,9 +1197,7 @@ impl PromPlanner { ) { Ok(v) => v, // The compiler refusing a shape is this route declining it. - Err(DataFusionError::NotImplemented(_)) => { - return Ok(declined("no_kernel_for_the_statement_shape")); - } + Err(DataFusionError::NotImplemented(_)) => return Ok(None), Err(e) => return Err(e).context(DataFusionPlanningSnafu), }; if compiled.inputs.len() != prepared.len() { @@ -1379,11 +1207,11 @@ impl PromPlanner { )) .context(DataFusionPlanningSnafu); } - for (index, (compiled_input, (scan, _, _, time, _, _, _, _))) in + for (index, (compiled_input, (scan, _, _, time, _, _, _))) in compiled.inputs.iter().zip(prepared.iter_mut()).enumerate() { if compiled_input.source_index != index { - return Ok(declined("compiled_sources_out_of_order")); + return Ok(None); } // The range this selector reads is the compiler's own: a window and an instant differ in which // side of the lookback or range their scan starts on, and an anchor can move the whole range @@ -1481,7 +1309,7 @@ impl PromPlanner { } let source_tags = prepared .iter() - .map(|(_, _, _, _, tags, _, _, _)| tags.clone()) + .map(|(_, _, _, _, tags, _, _)| tags.clone()) .collect::>(); // The complete runtime label set of each input: a scan bound to one metric names every series it // hands over, so `__name__` is one of its labels even though it is not a tag column. Rewriting the @@ -1489,7 +1317,7 @@ impl PromPlanner { // declares the name among its tags itself. let runtime_source_tags = prepared .iter() - .map(|(_, _, metric, _, tags, _, _, _)| { + .map(|(_, _, metric, _, tags, _, _)| { let mut tags = tags.clone(); if metric.is_some() && !tags.iter().any(|tag| tag == METRIC_NAME) { tags.push(METRIC_NAME.to_string()); @@ -1497,20 +1325,6 @@ impl PromPlanner { tags }) .collect::>(); - // The other half of what the canonical pass judges a dropped label by: the one metric each - // input is bound to. A scan of one table names every series it hands over, so `__name__` is - // the same for all of them and dropping it tells none of them apart; a discovered selector - // that expands into several metrics binds no single name, which is the fact that keeps - // `rate` over it collected. - let runtime_source_metrics = prepared - .iter() - .map(|(_, _, metric, _, _, _, _, _)| metric.clone()) - .collect::>(); - // The drop's other half, and the one no tree can answer about itself: whether every input - // keys its series by columns that always decode a value. One input that cannot say so - a - // nullable primary-key column, or a discovered selector whose metrics disagree - is enough - // for the pass to collect the drop rather than trust it. - let keys_survive = prepared.iter().all(|input| input.7); // Whether any input's table is split into regions along a partition column. Only the canonical // boundary claims a whole-query label-set answer, and a region-local producer cannot see the pieces // another region holds. @@ -1522,26 +1336,10 @@ impl PromPlanner { let tree = match promql::native::canonicalize::merge_label_rewrites( &compiled.tree, &runtime_source_tags, - &runtime_source_metrics, - keys_survive, ) { Ok(tree) => tree, - Err(_) => return Ok(declined("label_rewrites_do_not_merge")), + Err(_) => return Ok(None), }; - // What those collections cost: a node that drops the metadata labels is proved by its labels - // and its scan's key columns together now, so a drop the labels alone would have proved is - // collected exactly when the keys do not survive it - the pair a primary-key column can - // spell as `NULL` in one series and `''` in another. Counting the statements collected for - // that reason is what keeps the price of the fact observable. - if promql::native::plan::leaves_a_drop_to_the_storage_keys( - &tree, - &runtime_source_tags, - &runtime_source_metrics, - keys_survive, - ) { - SKIPPED_LABELSET_CHECK - .with(|skipped| skipped.set(Some(SKIPPED_LABELSET_CHECK_NULLABLE_KEY))); - } let root = tree.root(); // The placement facts the split and the segmentation below are chosen from, and the same // facts that say whether one node can run over the scans at all. @@ -1581,13 +1379,13 @@ impl PromPlanner { .collect::>(); let per_source = prepared .iter() - .map(|(_, _, metric, time_index, tags, _, _, _)| { + .map(|(_, _, metric, time_index, tags, _, _)| { (metric.clone(), time_index.clone(), tags.clone()) }) .collect::>(); let children = prepared .into_iter() - .map(|(scan, schema, metric, _, tags, _, split_series, _)| { + .map(|(scan, schema, metric, _, tags, _, split_series)| { // A discovered selector already carries its own sources: a single metric is one // `RawSeriesSource`, several are a `Union` of them, and each names the metric it // reads. Wrapping either again would nest a source in a source. @@ -1643,35 +1441,38 @@ impl PromPlanner { // A tree with no canonical merge claims no label-set answer, so it is untouched: a // per-series query on a partitioned table keeps fusing. if !spans_regions && any_partitioned && contains_canonical_merge(root) { - return Ok(declined("canonical_merge_over_partitioned_sources")); + return Ok(None); } // The kind the tree's root answers with, which every output column follows. It is not the kind the // inputs carry: `count_over_time`, `histogram_count` and `timestamp` reduce a histogram to a // float, while a selector, a histogram-answering range function and a label rewrite pass it on. - // A canonical merge hands on the samples it was given, so an answer kind crosses it unchanged: - // a histogram answer keeps its frames over the boundary, and the router merges them with the - // same kernel it merges floats with. // A shape whose kind cannot be named keeps the existing plan rather than declaring a column the // first batch might not fill. let root_kind = match statement_kind { Some(FieldKind::Histogram) => { + // A histogram answer standing on a canonical merge would run through the canonical + // boundary carrying histogram frames, a pairing nothing exercises yet, so such a + // statement keeps the existing plan. Every merge counts, not only a root one. + if merge_answers_histogram(root, FieldKind::Histogram) { + return Ok(None); + } match root.output_field_kind(Some(FieldKind::Histogram)) { Some(root_kind) => root_kind, - None => return Ok(declined("root_declares_no_field_kind")), + None => return Ok(None), } } Some(FieldKind::Float) | None => FieldKind::Float, }; let frame_metadata = if format == FusedOutputFormat::SeriesFrame { let NativeType::Samples(schema) = &root.output else { - return Ok(declined("frame_operand_is_not_samples")); + return Ok(None); }; if schema.eval_grid.is_none() { - return Ok(declined("frame_operand_has_no_grid")); + return Ok(None); } // A segment publishes frames, so it reports no name column of its own. let Some(output_tags) = fused_output_tags(root, &source_tags) else { - return Ok(declined("root_has_no_output_labels")); + return Ok(None); }; Some((root.output.clone(), output_tags)) } else { @@ -1685,20 +1486,20 @@ impl PromPlanner { if let Some(kind @ FieldKind::Histogram) = statement_kind && ((multi_input && per_source_inputs) || aggregate_reads_histogram(root, kind)) { - return Ok(declined("histogram_aggregate_needs_state_this_route_lacks")); + return Ok(None); } // The split keeps its prefix per-series, which is why it declines an operation whose // answer depends on the order its input arrives in. A node over more than one input reads // its inputs' partitions in the order they became ready, so that refusal holds there too. if multi_input && depends_on_input_order(root) { - return Ok(declined("answer_depends_on_input_order")); + return Ok(None); } // The native output schema can name only compiled value fields, the time index and the tag // columns a source declares. A discovered source declares `__name__` because the name tells two // metrics with equal labels apart (#10106); a named source does not, so a tree reading more than // one such source cannot report the distinct groups and declines. if multi_input && groups_by_metric_name(root) { - return Ok(declined("grouping_by_metric_name_over_several_sources")); + return Ok(None); } // Catch the shape refusals before choosing native execution: errors raised during physical // planning cannot fall back. This includes unsupported aggregates and `topk`, `limitk` or @@ -1706,7 +1507,7 @@ impl PromPlanner { // tree. A plan split over regions commits to the fused node without the split or a segment // above it, so it checks itself the same way before that choice. if (multi_input || per_source_inputs) && !native_tree_supported(&tree) { - return Ok(declined("runtime_refuses_the_tree")); + return Ok(None); } // A tree that ends in a canonical merge answers one series per label set, and the pieces of one // set were spread over the scan's partitions by the rewrite that created them. Lowering that @@ -1759,7 +1560,7 @@ impl PromPlanner { if root_kind != FieldKind::Float && !(root_kind == FieldKind::Histogram && histogram_state) { - return Ok(declined("split_aggregate_state_cannot_carry_the_answer")); + return Ok(None); } // The row-value layout is what the split is validated against whichever format the // caller asked for: it names one column per value the partial state accumulates, @@ -1767,16 +1568,16 @@ impl PromPlanner { let Some(value_schema) = Self::fused_root_schema(&expression, &tree, &source_tags, &time, root_kind)? else { - return Ok(declined("root_has_no_value_columns")); + return Ok(None); }; // A frame operand also needs the evaluated layout its consumers decode, which is // the metadata derived above: a request that could not name one keeps the existing // plan rather than publishing rows to a frame consumer. if format == FusedOutputFormat::SeriesFrame && frame_metadata.is_none() { - return Ok(declined("frame_operand_needs_an_evaluated_layout")); + return Ok(None); } let Some(rawinput) = children.into_iter().next() else { - return Ok(declined("split_aggregate_has_no_input")); + return Ok(None); }; let publish_grid = Arc::clone(&grid); let node = plan_split_aggregate( @@ -1858,7 +1659,7 @@ impl PromPlanner { // contribution the consumer needs. Keep that consumer on the existing plan; a tree over more // than one input is not this shape. if required_placement && !multi_input { - return Ok(declined("cross_series_consumer_over_one_scan")); + return Ok(None); } } // The tags are asked for before the fields, because the walk answers for a tree the @@ -1869,14 +1670,12 @@ impl PromPlanner { NativeType::Scalar(_) => Vec::new(), NativeType::Samples(_) => match fused_output_tags(tree.root(), &source_tags) { Some(output_tags) => with_row_name_column(tree.root(), output_tags), - None => return Ok(declined("root_has_no_output_labels")), + None => return Ok(None), }, - NativeType::AggregateState(_) => { - return Ok(declined("aggregate_state_is_not_a_statement_answer")); - } + NativeType::AggregateState(_) => return Ok(None), }; let Some(root_fields) = native_root_fields(tree.root()) else { - return Ok(declined("root_has_no_value_columns")); + return Ok(None); }; let output_schema = match format { FusedOutputFormat::PlainRows => { @@ -2015,18 +1814,8 @@ impl PromPlanner { fn empty_discovered_input(time_index: String) -> Result { let schema = empty_discovered_series_schema(); let input = empty_discovered_frame(&schema)?; - // No table stands behind this input, so no region splits it along a partition column - and no - // metadata states what its keys would survive, which is the answer that collects. - Ok(( - input, - schema, - None, - time_index, - Vec::new(), - false, - false, - false, - )) + // No table stands behind this input, so no region splits it along a partition column. + Ok((input, schema, None, time_index, Vec::new(), false, false)) } fn empty_discovered_frame(schema: &SeriesSchema) -> Result { @@ -3446,9 +3235,8 @@ mod tests { fused_output_tags(tree.root(), &columns), "a written name is a column the output publishes" ); - let collected = - promql::native::canonicalize::merge_label_rewrites(&tree, &runtime, &[], true) - .expect("the pass produces a legal tree"); + let collected = promql::native::canonicalize::merge_label_rewrites(&tree, &runtime) + .expect("the pass produces a legal tree"); assert!( matches!(collected.root().expr, NativeExpr::Merge { .. }), "a name rewrite writes an existing runtime label: {collected}" @@ -3495,17 +3283,16 @@ mod tests { ]]; let tree = compiled_test_tree(r#"label_replace(abs(x), "__name__", "same", "missing", "(.*)")"#); - let collected = - promql::native::canonicalize::merge_label_rewrites(&tree, &runtime, &[], true) - .expect("the pass produces a legal tree"); + let collected = promql::native::canonicalize::merge_label_rewrites(&tree, &runtime) + .expect("the pass produces a legal tree"); assert!( matches!(collected.root().expr, NativeExpr::Merge { .. }), "the merge sits above the name rewrite: {collected}" ); assert_eq!( - 2, + 1, collected.to_string().matches("Merge").count(), - "the name rewrite and the name drop below it are each collected: {collected}" + "exactly one merge: {collected}" ); // The rewrite writes a name over a dropped one, so the output publishes a name column. assert_eq!( @@ -3516,14 +3303,11 @@ mod tests { // A fresh ordinary destination above the same kernel is still uncollected: no scan can carry // an ordinary label it does not declare. let fresh = compiled_test_tree(r#"label_replace(abs(x), "dst", "$1", "host", "(.*)")"#); - let collected = - promql::native::canonicalize::merge_label_rewrites(&fresh, &runtime, &[], true) - .expect("the pass produces a legal tree"); - assert_eq!( - 1, - collected.to_string().matches("Merge").count(), - "the fresh ordinary destination adds no collection of its own, and `abs` keeps the one \ - its name drop needs: {collected}" + let collected = promql::native::canonicalize::merge_label_rewrites(&fresh, &runtime) + .expect("the pass produces a legal tree"); + assert!( + !collected.to_string().contains("Merge"), + "a fresh ordinary destination does not alias: {collected}" ); } diff --git a/src/query/src/promql/planner/test.rs b/src/query/src/promql/planner/test.rs index 3d092c670a4..e876d6e06bf 100644 --- a/src/query/src/promql/planner/test.rs +++ b/src/query/src/promql/planner/test.rs @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::promql::planner::native::{ - take_native_attempts, take_reported_decline, take_reported_skipped_labelset_check, -}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -524,7 +521,6 @@ async fn plan_direct_or( ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; planner .or_operator( @@ -615,7 +611,6 @@ async fn mixed_direct_or(histogram_on_left: bool) -> (PromPlanner, LogicalPlan) ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let left_context = direct_or_context("lhs", &["job", "k"], "v"); let right_context = direct_or_context("rhs", &["job", "k"], "v"); @@ -689,7 +684,6 @@ async fn mixed_aggregate_input(histograms: Vec) -> (PromPlanner }, promql_annotations: None, native_fused: false, - native_binary_offer: true, }; (planner, plan) } @@ -1377,66 +1371,6 @@ async fn native_engine_temporality_marker_provider() -> DfTableSourceProvider { ) } -/// One Mito table of one float field whose only tag is a nullable primary-key column. -/// -/// A metric-engine table is built with nullable tags, so one series of such a table can hold `NULL` -/// in the tag where another holds `''` - the pair a metadata drop normalizes into one label set, and -/// what the canonical pass collects a drop for. The statements that pay for it are counted, which -/// `a_nullable_key_statement_records_the_collection_it_pays_for` and -/// `a_rate_chain_over_a_nullable_key_pays_one_merge` read. -async fn build_test_table_provider_with_nullable_tag(table_name: &str) -> DfTableSourceProvider { - let catalog_list = MemoryCatalogManager::with_default_setup(); - let schema = Arc::new(Schema::new(vec![ - ColumnSchema::new( - "tag_0".to_string(), - ConcreteDataType::string_datatype(), - true, - ), - ColumnSchema::new( - "timestamp".to_string(), - ConcreteDataType::timestamp_millisecond_datatype(), - false, - ) - .with_time_index(true), - ColumnSchema::new( - "field_0".to_string(), - ConcreteDataType::float64_datatype(), - true, - ), - ])); - let table_meta = TableMetaBuilder::empty() - .schema(schema) - .engine(NATIVE_ENGINE.to_string()) - .primary_key_indices(vec![0]) - .value_indices(vec![2]) - .next_column_id(1024) - .build() - .unwrap(); - let table_info = TableInfoBuilder::default() - .name(table_name.to_string()) - .meta(table_meta) - .build() - .unwrap(); - assert!( - catalog_list - .register_table_sync(RegisterTableRequest { - catalog: DEFAULT_CATALOG_NAME.to_string(), - schema: DEFAULT_SCHEMA_NAME.to_string(), - table_name: table_name.to_string(), - table_id: 1024, - table: EmptyTable::from_table_info(&table_info), - }) - .is_ok() - ); - DfTableSourceProvider::new( - catalog_list, - false, - QueryContext::arc(), - DummyDecoder::arc(), - false, - ) -} - #[allow(clippy::too_many_arguments)] async fn build_test_table_provider_with_options( table_name_tuples: &[(String, String)], @@ -1523,35 +1457,21 @@ async fn build_test_native_histogram_table_provider(table_name: &str) -> DfTable build_test_native_histogram_table_provider_with_marker(table_name, false).await } -/// The same table with tag columns that always decode a value. -/// -/// A metric-engine table is built with nullable tags, so a metadata drop over it is collected for the -/// `NULL`/`''` pair a drop normalizes into one label set -/// ([`a_nullable_key_column_answers_a_histogram_statement_over_a_canonical_boundary`]). This fixture -/// states the other table: one whose keys always decode a value, so the labels alone prove the drop -/// and nothing has to be collected. -async fn build_test_native_histogram_table_provider_with_stable_keys( - table_name: &str, -) -> DfTableSourceProvider { - build_test_native_histogram_tables_with_marker(&[table_name], false, true).await -} - /// Several Mito tables of native histograms, for the multi-input gates. async fn build_test_native_histogram_tables(table_names: &[&str]) -> DfTableSourceProvider { - build_test_native_histogram_tables_with_marker(table_names, false, false).await + build_test_native_histogram_tables_with_marker(table_names, false).await } async fn build_test_native_histogram_table_provider_with_marker( table_name: &str, temporality_marker: bool, ) -> DfTableSourceProvider { - build_test_native_histogram_tables_with_marker(&[table_name], temporality_marker, false).await + build_test_native_histogram_tables_with_marker(&[table_name], temporality_marker).await } async fn build_test_native_histogram_tables_with_marker( table_names: &[&str], temporality_marker: bool, - stable_keys: bool, ) -> DfTableSourceProvider { let catalog_list = MemoryCatalogManager::with_default_setup(); let mut columns = vec![ @@ -1560,12 +1480,10 @@ async fn build_test_native_histogram_tables_with_marker( ConcreteDataType::string_datatype(), false, ), - // `le` is a nullable tag on a metric-engine table, which is what a drop over it has to be - // proved against; `stable_keys` states the table whose key columns always decode a value. ColumnSchema::new( LE_COLUMN_NAME.to_string(), ConcreteDataType::string_datatype(), - !stable_keys, + true, ), ]; if temporality_marker { @@ -2777,33 +2695,6 @@ async fn native_absent_over_time_plans_over_discovered_metrics() { ); } -/// Whether any canonical merge in `plan` asks to hand histogram frames across a boundary. -/// -/// A merge hands on the samples it was given, so its answer kind is its child's - which is the fact -/// the admission gate this route used to carry read, and the reason it kept such a statement on the -/// existing plan. This reads the same fact off the plan the route now builds, so a test says -/// "histogram frames cross this boundary" about the node that carries them rather than about the -/// statement's answer kind. -fn fused_merges_answer_histograms(plan: &LogicalPlan) -> bool { - fn visit(plan: &LogicalPlan, found: &mut bool) { - if let LogicalPlan::Extension(Extension { node }) = plan - && let Some(fused) = node.as_any().downcast_ref::() - { - *found |= fused.tree().root().any(&|node| { - matches!(node.expr, NativeExpr::Merge { .. }) - && node.output_field_kind(Some(FieldKind::Histogram)) - == Some(FieldKind::Histogram) - }); - } - for input in plan.inputs() { - visit(input, found); - } - } - let mut found = false; - visit(plan, &mut found); - found -} - /// Every native tree the plan carries, outermost first: one per stage the plan splits into. fn fused_trees(plan: &LogicalPlan) -> Vec { fn visit(plan: &LogicalPlan, trees: &mut Vec) { @@ -3343,176 +3234,6 @@ async fn native_fused_rate_plans_to_a_fused_series_node() { ); } -/// A single-metric `rate` streams: the window drops the metric name, but every series the scan -/// hands over is named the same one, so the drop tells none of them apart. Nothing has to collect -/// what the window hands out, and the plan keeps the one node over the scan. -#[tokio::test] -async fn native_fused_single_metric_rate_streams_without_a_canonical_merge() { - let query = "rate(some_metric[5m])"; - let plan = build_native_fused_plan(query, true).await; - let fused = find_fused_series(&plan).expect("the native route answers the query"); - assert_eq!( - Some("some_metric"), - fused.binding().metric_name.as_deref(), - "the binding names the one metric the scan reads" - ); - assert!( - !fused.is_canonical_producer(), - "`{query}` needs no canonical boundary: {plan}" - ); - assert!( - matches!(fused.tree().root().expr, NativeExpr::Window { .. }), - "`{query}`: the root stays the window: {:?}", - fused.tree().root().expr - ); - let [child] = fused.children() else { - panic!("`{query}` reads one input: {plan}"); - }; - assert!( - matches!(child, LogicalPlan::TableScan(_)), - "`{query}`: the window reads the scan itself: {plan}" - ); -} - -/// A `rate` over a selector that discovered several metrics keeps its collection: two of those -/// metrics can agree on every ordinary label, so the name the window drops is the only thing -/// telling them apart - and the merge is what reports the collision. -#[tokio::test] -async fn native_fused_multi_metric_rate_keeps_its_canonical_merge() { - let query = r#"rate({tag_0="x"}[5m])"#; - let plan = build_native_fused_plan_with_provider( - query, - true, - metric_discovery_provider(&["alpha_total", "beta_total"]).await, - ) - .await; - let consumer = - find_fused_series(&plan).unwrap_or_else(|| panic!("`{query}` is not native: {plan}")); - let [child] = consumer.children() else { - panic!("`{query}`: the boundary has one producer: {plan}"); - }; - let producer = find_fused_series(child).unwrap_or_else(|| panic!("`{query}`: {plan}")); - assert!(producer.is_canonical_producer(), "`{query}`: {plan}"); - assert!( - matches!(producer.tree().root().expr, NativeExpr::Merge { .. }), - "`{query}`: the window's pieces are collected: {:?}", - producer.tree().root().expr - ); -} - -/// A statement whose scan keys its series by a nullable column is collected for that column, and the -/// counter says so once per statement. -/// -/// The labels half of the drop's proof holds here: `rate(some_metric[5m])` over one metric proves -/// the name it drops is the one every series of the scan shares. The other half is the scan's key -/// columns, and this fixture's tag is a nullable primary key: one series can hold `NULL` there and -/// another `''`, which decode to no label once the drop removes the empty value with the name. The -/// pass therefore collects the drop - and reads the table to say so - which is what the count makes -/// observable. -/// -/// A window that keeps the name drops nothing, so it relies on nothing and counts nothing. -#[tokio::test] -async fn a_nullable_key_statement_records_the_collection_it_pays_for() { - let table_provider = build_test_table_provider_with_nullable_tag("some_metric").await; - let plan = - build_native_fused_plan_with_provider("rate(some_metric[5m])", true, table_provider).await; - let trees = fused_trees(&plan); - assert!( - !trees.is_empty(), - "`rate(some_metric[5m])` left the native route" - ); - assert!( - trees.iter().any(|tree| tree.contains("Merge")), - "the nullable key collected the window's drop: {trees:?}" - ); - assert_eq!( - Some("nullable_key"), - take_reported_skipped_labelset_check(), - "the drop was collected for the key columns and not counted: {}", - plan.display_indent_schema() - ); - - let table_provider = build_test_table_provider_with_nullable_tag("some_metric").await; - let plan = build_native_fused_plan_with_provider( - "last_over_time(some_metric[5m])", - true, - table_provider, - ) - .await; - assert!( - find_fused_series(&plan).is_some(), - "`last_over_time(some_metric[5m])` left the native route: {}", - plan.display_indent_schema() - ); - assert_eq!( - None, - take_reported_skipped_labelset_check(), - "a window that keeps the name collects nothing for the keys: {}", - plan.display_indent_schema() - ); -} - -/// A nullable key column costs a chain of metadata drops one merge, not one per drop. -/// -/// The merge sits above the window - the bottom-most drop - and everything above it reads one series -/// per canonical label set, so the drops above it need no collection of their own. A merge per drop -/// would buffer the whole input once per kernel for a pair only the bottom one can still fold. -#[tokio::test] -async fn a_rate_chain_over_a_nullable_key_pays_one_merge() { - for query in [ - "rate(some_metric[5m]) * 2", - "abs(abs(rate(some_metric[5m])))", - ] { - let table_provider = build_test_table_provider_with_nullable_tag("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let trees = fused_trees(&plan); - assert!( - !trees.is_empty(), - "`{query}` is not native: {}", - plan.display_indent_schema() - ); - let merges: usize = trees.iter().map(|tree| tree.matches("Merge").count()).sum(); - assert_eq!( - 1, merges, - "`{query}` pays for one merge over the nullable key: {trees:?}" - ); - } -} - -/// A scan that carries `__type__` or `__unit__` as a tag can hold two series that agree on -/// everything else, and a range function drops those two along with the name. A fixed metric name -/// proves nothing about them, so the collection stays. -#[tokio::test] -async fn native_fused_metadata_tag_in_the_binding_keeps_the_collection() { - for metadata in ["__type__", "__unit__"] { - let query = "rate(some_metric[5m])"; - let specs = vec![( - (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), - 2, - 1, - )]; - let tags = vec![vec!["tag_0".to_string(), metadata.to_string()]]; - let plan = build_native_fused_plan_with_provider( - query, - true, - build_test_table_provider_with_tsid_tag_names(&specs, &tags, None).await, - ) - .await; - let consumer = find_fused_series(&plan) - .unwrap_or_else(|| panic!("{metadata}: `{query}` is not native: {plan}")); - let [child] = consumer.children() else { - panic!("{metadata}: `{query}`: the boundary has one producer: {plan}"); - }; - let producer = - find_fused_series(child).unwrap_or_else(|| panic!("{metadata}: `{query}`: {plan}")); - assert!( - matches!(producer.tree().root().expr, NativeExpr::Merge { .. }), - "{metadata}: `{query}` keeps the collection: {:?}", - producer.tree().root().expr - ); - } -} - #[tokio::test] async fn native_fused_time_partitioned_table_reads_through_a_raw_source() { let table_provider = native_engine_provider(vec![1]).await; @@ -4090,130 +3811,6 @@ async fn native_fused_metric_engine_two_aggregate_pair_is_one_execution() { } } -/// Every statement is offered to the native route once, and a statement the flag leaves alone is -/// offered none. -/// -/// The count is what bounds planning: one offer compiles the whole tree below it, so a route that -/// re-offered a subtree for every operator below it would cost the square of the expression's size. -#[tokio::test] -async fn native_offer_is_counted_once_per_statement() { - let _ = take_native_attempts(); - let plan = build_native_fused_plan("rate(some_metric[5m])", true).await; - assert!(find_fused_series(&plan).is_some(), "{plan}"); - assert_eq!( - 1, - take_native_attempts(), - "the statement is offered exactly once" - ); - - let plan = build_native_fused_plan("rate(some_metric[5m])", false).await; - assert!(find_fused_series(&plan).is_none(), "{plan}"); - assert_eq!( - 0, - take_native_attempts(), - "the flag off offers nothing to the native route" - ); -} - -/// A propagated matcher is offered once for the operand it rewrote, and nothing below it. -/// -/// `matching_filters::propagate` writes a matcher into an operand's scan, which makes that operand -/// an expression the offer spent at its parent never compiled: that offer read the statement as it -/// was written, and the matcher reached the scan afterwards. The rewrite is therefore offered of -/// its own, and the offer it is handed is spent again by the rewritten operand's own subtree, -/// exactly as this one was. So `N` operators below the rewrite buy one offer - not `N`, and not the -/// `N` positions times `N` subtrees an unconditional re-offer would compile. -#[tokio::test] -async fn native_offer_counts_a_propagated_operand_once() { - // The same match with the rewritten operand two operators deep and six deep: the matcher - // crosses in both, and the four extra operators are the operand's subtree rather than more - // expressions to offer. - let shallow = "((some_metric * 2) * 2) / on(tag_0) \ - some_metric{tag_0=\"x\",__field__=\"field_0\"}"; - let deep = "((((((some_metric * 2) * 2) * 2) * 2) * 2) * 2) / on(tag_0) \ - some_metric{tag_0=\"x\",__field__=\"field_0\"}"; - let mut counts = Vec::new(); - for query in [shallow, deep] { - let _ = take_native_attempts(); - let plan = build_native_fused_plan(query, true).await; - let display = plan.display_indent().to_string(); - // The count below is only a bound if the matcher really crossed to the other operand. - assert!( - display.contains("some_metric.tag_0 = Utf8(\"x\")"), - "{query}: the matcher reached the other operand's scan: {display}" - ); - counts.push(take_native_attempts()); - } - // Seven offers either way: the statement, its root's own retry under the legacy-only - // `__field__` matcher, the frame operands the composed attempts beside those two read, and the - // propagated operand itself. The four operators that make the deep statement deeper are inside - // that operand's subtree, so they buy no offer of their own - an offer that was not spent again - // by the rewrite would compile each of them, and the deep statement would cost four more. - assert_eq!( - (7, 7), - (counts[0], counts[1]), - "a propagated operand is offered once whatever its depth: seven offers at both depths" - ); -} - -/// The same matcher-propagating match, with its rewritten operand `depth` operators deep. -fn propagation_query(depth: usize) -> String { - let operand = (0..depth).fold("some_metric".to_string(), |inner, _| { - format!("({inner} * 2)") - }); - format!("{operand} / on(tag_0) some_metric{{tag_0=\"x\",__field__=\"field_0\"}}") -} - -/// How a statement of [`propagation_query`] plans: whether a native node answers the rewritten -/// operand, and whether the propagated matcher reached the other operand's scan. -async fn propagation_route(depth: usize) -> (bool, bool) { - let plan = build_native_fused_plan(&propagation_query(depth), true).await; - let display = plan.display_indent().to_string(); - ( - find_fused_series(&plan).is_some(), - display.contains("some_metric.tag_0 = Utf8(\"x\")"), - ) -} - -/// A chain under a propagated matcher plans on the stack a query worker has, however deep it is. -/// -/// The planner recurses once per operator and charges every level's frame to the stack of the level -/// above it, so a chain of constant multiplications deep enough - eight operators was enough on a -/// two-megabyte thread before the descent grew its own stack - aborted the process rather than -/// answering. nextest's own test thread is not a worker's, so the depths are planned on a thread -/// this test sizes like one. -#[test] -fn a_deep_propagated_operand_plans_within_a_worker_stack() { - let route = std::thread::Builder::new() - .name("deep_propagated_operand".to_string()) - .stack_size(2 * 1024 * 1024) - .spawn(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("a current-thread runtime"); - runtime.block_on(async { - let shallow = propagation_route(2).await; - for depth in [8, 10, 12] { - assert_eq!( - shallow, - propagation_route(depth).await, - "an operand {depth} operators deep plans as one 2 operators deep does" - ); - } - shallow - }) - }) - .expect("a thread with a worker's stack") - .join() - .expect("operands 2, 8, 10 and 12 operators deep plan on a two-megabyte stack"); - assert_eq!( - (true, true), - route, - "the rewritten operand is native and the matcher reached the other operand's scan" - ); -} - /// A metric table without logical-table metadata declines to the legacy plan. #[tokio::test] async fn native_fused_metric_table_without_logical_metadata_falls_back() { @@ -4305,12 +3902,6 @@ async fn native_fused_engine_without_raw_series_falls_back() { without_flag.display_indent().to_string(), "a file-engine table must keep the existing plan" ); - // The refusal is reported with its cause, not a bare Ok(None). - assert_eq!( - take_reported_decline(), - Some("engine_without_raw_series_scanner"), - "the refusal names the engine's missing raw-series scanner" - ); } /// A value field the raw-series readers cannot serve keeps the existing plan. @@ -5316,10 +4907,8 @@ async fn native_composed_binary_declines_to_the_existing_plan() { ); // An operand that is not a frame leaves the statement's own wrapper in place: the pair is - // refused, so the existing plan answers the statement. The offer is the statement's, and the - // binary that makes it spends it for the whole subtree below: the operand is planned by the - // existing path too, because a refused chain of n operators would otherwise compile its - // subtrees again at every operator. + // refused, so the existing plan answers the statement while the operand that does compose keeps + // its own answer below it. for query in [ // A scalar root is not a frame: composing it by vector matching would broadcast it as if it // were a one-series vector, which is not what the query asked for. @@ -5327,7 +4916,7 @@ async fn native_composed_binary_declines_to_the_existing_plan() { // Neither is a scalar function, however it is written. "(hist * on(tag_0) floats) / time()", // A comparison between a histogram and a float has no kernel at this pair either, so the - // whole statement stops where the pair was refused. + // operand's own answer stops where it was admitted. "(hist * on(tag_0) floats) > on(tag_0) floats", ] { let plan = @@ -5339,22 +4928,8 @@ async fn native_composed_binary_declines_to_the_existing_plan() { "`{query}` has no composed answer: {display}" ); assert!( - find_fused_series(&plan).is_none(), - "`{query}` spends its one offer above the operand: {display}" - ); - let legacy = PromPlanner::stmt_to_plan_with_annotations( - native_mixed_kind_provider().await, - &build_eval_stmt(query), - &build_query_engine_state(), - None, - false, - ) - .await - .unwrap_or_else(|error| panic!("`{query}`: the existing route must answer it: {error}")); - assert_eq!( - legacy.display_indent().to_string(), - display, - "`{query}` must keep the existing plan" + find_fused_series(&plan).is_some(), + "`{query}` keeps the operand's own answer: {display}" ); } } @@ -6264,6 +5839,7 @@ async fn native_fused_pointwise_wrapper_around_a_pair_is_admitted() { "1 - some_metric / some_metric", "(some_metric / some_metric) * 100", "abs(some_metric - on(tag_0) some_metric)", + "(some_metric > some_metric) * 100", ] { let plan = build_native_fused_plan(query, true).await; let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}")); @@ -6277,44 +5853,6 @@ async fn native_fused_pointwise_wrapper_around_a_pair_is_admitted() { assert_eq!(2, fused.inputs().len(), "{query}"); } - // A filtering comparison keeps the metric name, so it hands its series over under a label set - // that still tells them apart; the arithmetic broadcast above it drops that name, and the drop - // is only injective when the name it removes is one value over every series below it. The label - // walk cannot prove that across a match - a match reports its operands' labels, not one - // metric's - so the broadcast is collected, and the collection turns the statement into a - // canonical boundary over one segment that reads each occurrence through its own source. - let query = "(some_metric > some_metric) * 100"; - let plan = build_native_fused_plan(query, true).await; - let consumer = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}")); - assert!( - matches!(consumer.tree().root().expr, NativeExpr::Input(_)), - "{query}: the consumer reads the collected boundary: {plan}" - ); - let [child] = consumer.children() else { - panic!("{query}: the boundary has one producer: {plan}"); - }; - let producer = find_fused_series(child).unwrap_or_else(|| panic!("{query}: {plan}")); - assert!(producer.is_canonical_producer(), "{query}: {plan}"); - assert!( - matches!(producer.tree().root().expr, NativeExpr::Merge { .. }), - "{query} collects the broadcast's name drop: {:?}", - producer.tree().root().expr - ); - // The merge collects what the two occurrences produced, so the work below the boundary is one - // segment reading each occurrence through its own source. - let [segment] = producer.children() else { - panic!("{query}: the producer reads one segment: {plan}"); - }; - let segment = find_fused_series(segment).unwrap_or_else(|| panic!("{query}: {plan}")); - assert_eq!(2, segment.inputs().len(), "{query}: {plan}"); - for input in segment.inputs() { - assert!( - matches!(input, LogicalPlan::Extension(extension) - if extension.node.as_any().is::()), - "{query}: each occurrence reads its own source: {plan}" - ); - } - let query = "ceil(some_metric)"; let plan = build_native_fused_plan(query, true).await; assert!(find_fused_series(&plan).is_some(), "{query}: {plan}"); @@ -7303,7 +6841,7 @@ async fn native_fused_limitk_over_a_windowed_aggregate_keeps_its_shape() { ); assert_eq!( Some(true), - unifies_its_own_contributions(window_node, &[], &[]), + unifies_its_own_contributions(window_node), "`{query}`: the window emits one series per label set, so `limitk` collects nothing" ); let NativeExpr::Aggregate { @@ -10003,7 +9541,6 @@ async fn comparison_binary_join_uses_tsid_and_keeps_it_in_filtered_result() { ctx: PromPlannerContext::from_eval_stmt(&eval_stmt).unwrap(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let plan = planner .prom_expr_to_plan(&eval_stmt.expr, &build_query_engine_state()) @@ -10998,8 +10535,7 @@ async fn an_aggregate_over_an_unnamed_kind_keeps_the_existing_plan() { "sum(some_metric or some_metric)", "sum(some_metric and some_metric)", ] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); assert!( @@ -11025,30 +10561,19 @@ async fn an_aggregate_over_an_unnamed_kind_keeps_the_existing_plan() { pending.extend(node.inputs()); } } - // A float-producing window still answers on the route: its count, not the input histogram, - // reaches the fold. The name `count_over_time` drops is the one every series of a single-metric - // scan shares, and these tables' key columns always decode a value, so the labels are the whole - // proof and nothing collects the prefix - and a prefix nothing collects is exactly what the - // split folds: the statement runs as a region-side partial aggregate and a final fold above it. - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + // A float-producing window still permits the aggregate itself to split natively: its count, + // not the input histogram, reaches the fold. + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider( "sum(count_over_time(some_metric[5m]))", true, table_provider, ) .await; - let display = plan.display_indent_schema().to_string(); - let partial = find_fused_series(&plan) - .unwrap_or_else(|| panic!("the float prefix left the native route: {display}")); assert!( find_final_split(&plan).is_some(), - "the uncollected prefix did not split: {display}" - ); - assert!( - matches!(partial.tree().root().expr, NativeExpr::Aggregate { .. }), - "the state-exporting producer folds: {}\n{display}", - partial.tree() + "the float prefix no longer splits natively: {}", + plan.display_indent_schema() ); } @@ -11056,136 +10581,81 @@ async fn an_aggregate_over_an_unnamed_kind_keeps_the_existing_plan() { /// /// The canonicalization pass wraps a collapsed rewrite in a merge, which the kind walk used to /// leave undescribed. "Cannot name it" then read as "possibly a histogram", so -/// `label_replace(histogram_count(h), ...)` kept the existing plan even though it answers floats. -/// The walk now names a merge (it concatenates samples of one field) and a set operation (its sides -/// agree on the kind), so that statement stays on the route - while a fold above the collection the -/// same rewrite's input needs, and a histogram that really does enter an aggregate or stand on a -/// canonical merge, still keep the existing plan. +/// `label_replace(histogram_count(h), ...)` and its aggregated form kept the existing plan even +/// though both answer floats. The walk now names a merge (it concatenates samples of one field) +/// and a set operation (its sides agree on the kind), so those statements stay on the route while +/// a histogram that really does enter an aggregate still keeps the existing plan. #[tokio::test] async fn a_label_rewrite_keeps_a_float_answer_on_the_route() { - let query = "label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\")"; - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let fused = find_fused_series(&plan) - .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); - // The value column is resolved by its type: the expression names it, so the compiled field - // name is not what a reader can address. - let field = native_value_field(fused, &format!("{query}: {display}")); - assert_eq!( - &ArrowDataType::Float64, - field.data_type(), - "{query} declared a histogram column for a float answer: {display}" - ); - // A histogram answer standing on the same collection crosses it: the merge carries histogram - // frames, and where the statement answers a float the map reducing them sits above the boundary. - // Both orderings are admitted because the merge only concatenates one field's samples - what it - // carries is its child's business - so each of these names the kind it publishes. - for (query, histogram) in [ - ( - "label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")", - true, - ), - ( - "histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\"))", - false, - ), - ( - "sum(histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")))", - false, - ), + for query in [ + "label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\")", + "sum(label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\"))", ] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan) .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); + let field = native_value_field(fused, &format!("{query}: {display}")); assert_eq!( - if histogram { - native_histogram_value_type().as_arrow_type() - } else { - ArrowDataType::Float64 - }, - *native_value_field(fused, &format!("{query}: {display}")).data_type(), - "{query} declared the wrong column kind: {display}" + &ArrowDataType::Float64, + field.data_type(), + "{query} declared a histogram column for a float answer: {display}" ); - let trees = fused_trees(&plan); + } + // A histogram answer standing on a canonical merge is out of this route's reach for now: the + // canonical boundary carrying histogram frames is the follow-up's subject. Both orderings + // count - the merge at the root, and a merge below a map that answers floats - because a merge + // anywhere in the tree is a boundary that would carry histograms. + for query in [ + "label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")", + "histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\"))", + "sum(histogram_count(label_replace(some_metric, \"tag_0\", \"x\", \"tag_0\", \".*\")))", + ] { + let table_provider = build_test_native_histogram_table_provider("some_metric").await; + let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; assert!( - trees - .iter() - .any(|tree| tree.contains("Merge: canonical label-set merge")), - "{query} answers without the merge the rewrite needs: {trees:?}" + find_fused_series(&plan).is_none(), + "{query} took a histogram answer through the canonical boundary: {}", + plan.display_indent_schema() ); } - // The map inside the rewrite answers a float, so the one collection left is the rewrite's own, - // and a fold above that merge does not carry a histogram: the name `histogram_count` drops is - // the one every series of a single-metric scan shares, which the labels alone prove. The - // statement therefore stays on the route, where it was collected twice before. - let query = - "sum(label_replace(histogram_count(some_metric), \"tag_0\", \"x\", \"tag_0\", \".*\"))"; - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let fused = find_fused_series(&plan) - .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); - let field = native_value_field(fused, &format!("{query}: {display}")); - assert_eq!( - &ArrowDataType::Float64, - field.data_type(), - "{query} declared a histogram column for a float answer: {display}" - ); } #[tokio::test] async fn a_segmented_parent_answers_the_kind_of_the_uncut_root() { - // Both operands answer floats, so the parent's column is the float one whatever the segments - // below it read: a kind taken from the scan's histogram field would declare the wrong column. - for query in [ - "count_over_time(some_metric[5m]) + on (__name__) count_over_time(some_metric[5m])", - "histogram_count(some_metric) + on (__name__) histogram_count(some_metric)", + for (query, histogram) in [ + ( + "count_over_time(some_metric[5m]) + on (__name__) count_over_time(some_metric[5m])", + false, + ), + ( + "histogram_count(some_metric) + on (__name__) histogram_count(some_metric)", + false, + ), + // The same segmentation over a tree that does answer histograms keeps the histogram + // column, which is what tells the two derivations apart: a kind forced to `Float` would + // pass the two cases above and fail this one. + ("some_metric * 2 + on (__name__) some_metric * 2", true), ] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan).unwrap_or_else(|| { panic!("{query} left the native route instead of segmenting: {display}") }); let field = native_value_field(fused, &format!("{query}: {display}")); + let expected = if histogram { + native_histogram_value_type().as_arrow_type() + } else { + ArrowDataType::Float64 + }; assert_eq!( - &ArrowDataType::Float64, + &expected, field.data_type(), "{query} declared the wrong column kind: {display}" ); } - // A histogram answer through the same match stays on the route as long as nothing collects it: - // `some_metric * 2` drops the metadata labels over one metric, which the labels prove, so no - // merge carries histogram frames and the gate that refuses one is never reached. - let query = "some_metric * 2 + on (__name__) some_metric * 2"; - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - assert!( - find_fused_series(&plan).is_some(), - "{query} left the native route although nothing collects it: {display}" - ); - let mut pending = vec![&plan]; - while let Some(node) = pending.pop() { - if let LogicalPlan::Extension(Extension { node }) = node - && let Some(fused) = node.as_any().downcast_ref::() - { - assert!( - !matches!(fused.tree().root().expr, NativeExpr::Merge { .. }), - "{query} collects a histogram answer: {}\n{display}", - fused.tree() - ); - } - pending.extend(node.inputs()); - } } #[tokio::test] @@ -11205,157 +10675,6 @@ async fn native_histogram_selector_plans_a_native_producer() { ); } -/// A nullable key column collects a histogram statement onto a canonical boundary. -/// -/// A metric-engine table is built with nullable tag columns, so a metadata drop over one is collected -/// for the `NULL`/`''` pair it normalizes into one label set - and for a histogram answer that -/// collection is a canonical merge carrying histogram frames. It is the boundary a float answer gets: -/// a producer running the statement's own tree per partition, the router collecting one label set -/// from every partition with the kernel that merges floats as it merges histograms, and a consumer -/// answering the histogram column above it. The same statement over a table whose key columns always -/// decode a value is proved by its labels, so nothing is collected and the histogram answer is one -/// node. -#[tokio::test] -async fn a_nullable_key_column_answers_a_histogram_statement_over_a_canonical_boundary() { - for query in ["rate(some_metric[5m])", "some_metric * 2"] { - let table_provider = build_test_native_histogram_table_provider("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let fused = find_fused_series(&plan) - .unwrap_or_else(|| panic!("`{query}` left the route instead of answering: {display}")); - assert_eq!( - &native_histogram_value_type().as_arrow_type(), - native_value_field(fused, &format!("`{query}`: {display}")).data_type(), - "`{query}` declared something other than the histogram it answers: {display}" - ); - // The merge is the collection the drop is paid for, and it answers what its child answers: - // the statement's kind, histograms included. - let trees = fused_trees(&plan); - assert!( - trees - .iter() - .any(|tree| tree.contains("Merge: canonical label-set merge")), - "`{query}` answers without the merge its drop is collected into: {trees:?}" - ); - // What crosses the boundary is the histogram payload itself: the merge that collected the - // drop answers histograms, so the frames the producer hands over are histogram frames. - assert!( - fused_merges_answer_histograms(&plan), - "`{query}` collected a field that is not the histogram it answers: {display}" - ); - } - // Over a table whose keys always decode a value the labels alone prove the drop, so the answer - // is one node and nothing collects it. - for query in ["rate(some_metric[5m])", "some_metric * 2"] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let trees = fused_trees(&plan); - assert!( - find_fused_series(&plan).is_some(), - "`{query}`: the proved drop left the route: {display}" - ); - assert!( - !trees.iter().any(|tree| tree.starts_with("Merge")), - "`{query}` collected a proved drop: {trees:?}" - ); - } -} - -/// A float answer reduces the histograms a boundary carried, and the boundary stays below it. -/// -/// `histogram_count(rate(h[5m]))` answers a float, so nothing about its root kind says how the -/// window below it is carried: `rate` drops the metadata labels over a nullable key, so the window -/// is collected into a canonical merge that hands histogram frames across the boundary, and -/// `histogram_count` maps those histograms to floats above it. The frames the producer publishes are -/// still histogram frames - the payload struct is what says so - and the consumer answers the float -/// column. -#[tokio::test] -async fn a_float_answer_reduces_the_histograms_a_boundary_carried() { - let query = "histogram_count(rate(some_metric[5m]))"; - let table_provider = build_test_native_histogram_table_provider("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let fused = find_fused_series(&plan) - .unwrap_or_else(|| panic!("`{query}` left the route instead of answering: {display}")); - assert_eq!( - &ArrowDataType::Float64, - native_value_field(fused, &format!("`{query}`: {display}")).data_type(), - "`{query}` declared a histogram column for a float answer: {display}" - ); - let trees = fused_trees(&plan); - let boundary = trees - .iter() - .find(|tree| tree.contains("Merge: canonical label-set merge")) - .unwrap_or_else(|| panic!("`{query}` answers without the collected window: {trees:?}")); - // What the merge collects is the window that dropped the name, and the field it hands over is - // the histogram payload: the native frame schema carries the histogram struct's own columns. - assert!( - boundary.contains("Window: rate "), - "`{query}`: the boundary collects something other than the window: {boundary}" - ); - // The frames the boundary hands over hold histograms, which is what says the map above it reads - // histograms rather than floats handed to it in their place. - assert!( - fused_merges_answer_histograms(&plan), - "`{query}`: the boundary carries a field that is not a histogram: {display}" - ); - // The map that answers the float reads the collected frames, so it sits above the boundary. - let reducing = trees - .iter() - .position(|tree| tree.contains("histogram_count")) - .unwrap_or_else(|| panic!("`{query}` has no map that reduces the histograms: {trees:?}")); - let collecting = trees - .iter() - .position(|tree| tree.contains("Merge: canonical label-set merge")) - .expect("the boundary was found above"); - assert!( - reducing != collecting, - "`{query}`: the reduction is the boundary itself: {trees:?}" - ); -} - -/// A histogram answer that crosses a collected drop still segments. -/// -/// The tree's two operands each drop the metadata labels over a nullable key, so each is collected, -/// and the match above them reads a histogram from each side. Segmentation cuts the tree into -/// segments that run where their inputs are read and keeps the collected boundary as a frame stage, -/// exactly as it does for a float answer - so the histogram frames cross a boundary that is not the -/// statement's own root. -#[tokio::test] -async fn a_segmented_histogram_answer_crosses_its_collected_boundary() { - let query = "some_metric * 2 + on (__name__) some_metric * 2"; - let table_provider = build_test_native_histogram_table_provider("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let fused = find_fused_series(&plan) - .unwrap_or_else(|| panic!("`{query}` left the route instead of answering: {display}")); - assert_eq!( - &native_histogram_value_type().as_arrow_type(), - native_value_field(fused, &format!("`{query}`: {display}")).data_type(), - "`{query}` declared something other than the histogram it answers: {display}" - ); - let trees = fused_trees(&plan); - assert!( - trees.len() > 1, - "`{query}` answers in one node, so nothing was segmented: {trees:?}" - ); - let collected = trees - .iter() - .filter(|tree| tree.contains("Merge: canonical label-set merge")) - .collect::>(); - assert_eq!( - 2, - collected.len(), - "`{query}`: each operand pays for its own drop: {trees:?}" - ); - assert!( - fused_merges_answer_histograms(&plan), - "`{query}` collected a field that is not a histogram: {display}" - ); -} - /// A histogram fold is admitted below a pointwise operator. /// /// The gate cannot look at the root alone: `sum(h) * 2` has a binary root and the fold sits below @@ -11373,8 +10692,7 @@ async fn a_histogram_fold_is_admitted_below_a_pointwise_operator() { "avg(some_metric) + 1", "-sum(some_metric)", ] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let final_node = find_final_split(&plan) @@ -11474,11 +10792,13 @@ async fn presence_and_order_aggregates_fold_histograms_natively() { #[tokio::test] async fn histogram_statements_run_natively_until_an_aggregate_folds_them() { - // A bare scan drops nothing, and `last_over_time` keeps the metric name, so neither hands out a - // label set two of its series could meet under. - for query in ["some_metric", "last_over_time(some_metric[5m])"] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + for query in [ + "some_metric", + "rate(some_metric[5m])", + "sum_over_time(some_metric[5m])", + "some_metric * 2", + ] { + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; assert!( find_fused_series(&plan).is_some(), @@ -11486,28 +10806,6 @@ async fn histogram_statements_run_natively_until_an_aggregate_folds_them() { plan.display_indent_schema() ); } - // `rate`, `sum_over_time` and `some_metric * 2` all answer a histogram, and each drops the - // metadata labels over one metric - which the labels alone prove, whatever the scan's `le` - // column can hold. Nothing collects them, so no merge carries a histogram frame and the gate - // that refuses one is never reached: all three stay on the route. - for query in [ - "rate(some_metric[5m])", - "sum_over_time(some_metric[5m])", - "some_metric * 2", - ] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; - let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; - let display = plan.display_indent_schema().to_string(); - let fused = find_fused_series(&plan).unwrap_or_else(|| { - panic!("{query} left the native route although nothing collects it: {display}") - }); - assert!( - !matches!(fused.tree().root().expr, NativeExpr::Merge { .. }), - "{query} collects a histogram answer: {}\n{display}", - fused.tree() - ); - } // `rate` answers histograms, so an aggregate over it folds histograms - and that folds now // runs here, which is what `a_histogram_sum_or_avg_folds_natively` covers. } @@ -12267,7 +11565,6 @@ async fn mixed_native_histogram_rate_executes_real_ranges() { }, promql_annotations: Some(collector.clone()), native_fused: false, - native_binary_offer: true, }; let input = LogicalPlan::Extension(Extension { node: Arc::new( @@ -13758,7 +13055,6 @@ async fn native_scan_bounds_preserve_zero_lookback_and_overflow() { ctx: PromPlannerContext::from_eval_stmt(&build_eval_stmt("some_metric")).unwrap(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; planner.ctx.time_index_column = Some("timestamp".to_string()); planner.ctx.start = 1_000; @@ -14629,7 +13925,6 @@ async fn test_direct_or_numeric_value_types() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let left_context = direct_or_context("lhs", &["job"], "v"); let right_context = direct_or_context("rhs", &["job"], "v"); @@ -15864,7 +15159,6 @@ async fn test_mixed_or_value_aliases_do_not_replace_labels() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let left = LogicalPlanBuilder::from(scan(&left)) .project(vec![ @@ -16618,8 +15912,7 @@ async fn a_histogram_sum_or_avg_folds_natively() { "sum(rate(some_metric[5m]))", "avg by (tag_0) (some_metric)", ] { - let table_provider = - build_test_native_histogram_table_provider_with_stable_keys("some_metric").await; + let table_provider = build_test_native_histogram_table_provider("some_metric").await; let plan = build_native_fused_plan_with_provider(query, true, table_provider).await; let display = plan.display_indent_schema().to_string(); let final_node = find_final_split(&plan) @@ -16685,6 +15978,11 @@ async fn native_composed_binary_admits_a_source_free_frame_operand() { /// shape that did not would decline to the existing plan; neither may re-offer the same expression. #[tokio::test] async fn native_composed_binary_recursion_terminates_on_a_deep_expression() { + fn count_fused(plan: &LogicalPlan) -> usize { + usize::from(is_fused_series(plan)) + + plan.inputs().into_iter().map(count_fused).sum::() + } + let depth = 32; let mut query = "hist * on(tag_0) floats".to_string(); for _ in 1..depth { @@ -16701,80 +15999,9 @@ async fn native_composed_binary_recursion_terminates_on_a_deep_expression() { assert!(!display.contains("Join:"), "{depth}-deep: {display}"); // One composition per operator, plus the leaf frames below them. assert!( - count_fused_nodes(&plan) > depth, + count_fused(&plan) > depth, "{depth}-deep: {} fused nodes: {display}", - count_fused_nodes(&plan) - ); -} - -/// The same match over two field kinds, `depth` binary operators deep. -fn composition_query(depth: usize) -> String { - let mut query = "hist * on(tag_0) floats".to_string(); - for _ in 1..depth { - query = format!("({query}) / on(tag_0) floats"); - } - query -} - -fn count_fused_nodes(plan: &LogicalPlan) -> usize { - usize::from(is_fused_series(plan)) - + plan - .inputs() - .into_iter() - .map(count_fused_nodes) - .sum::() -} - -/// A deep native composition plans on the stack a query worker has. -/// -/// `try_plan_native` and `try_plan_native_composed_binary` call each other once per binary operator, -/// and every level's frames are charged to the stack of the level above it. A left-deep chain of -/// matches over two field kinds 64 operators deep aborted the process - a stack overflow, not a -/// declined plan - on the two megabytes a query worker has, before `try_plan_native` grew its stack -/// for the poll of its future; 56 operators deep still planned. nextest's own test thread is not a -/// worker's, so the depths here are planned on a thread this test sizes like one, and the deepest of -/// them is twice the depth that aborted. -#[test] -fn a_deep_native_composition_plans_within_a_worker_stack() { - let depths = [10, 24, 48, 64, 128]; - let planned = std::thread::Builder::new() - .name("deep_native_composition".to_string()) - .stack_size(2 * 1024 * 1024) - .spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("a current-thread runtime"); - runtime.block_on(async { - let mut planned = Vec::new(); - for depth in depths { - let query = composition_query(depth); - let plan = build_native_fused_plan_with_provider( - &query, - true, - native_mixed_kind_provider().await, - ) - .await; - let display = plan.display_indent().to_string(); - assert!(!display.contains("Join:"), "{depth}-deep: {display}"); - // One composition per operator, plus the leaf frames below them. - assert!( - count_fused_nodes(&plan) > depth, - "{depth}-deep: {} fused nodes: {display}", - count_fused_nodes(&plan) - ); - planned.push(depth); - } - planned - }) - }) - .expect("a thread with a worker's stack") - .join() - .expect("compositions 10, 24, 48, 64 and 128 operators deep plan on a two-megabyte stack"); - assert_eq!( - depths.to_vec(), - planned, - "every depth plans as a native composition" + count_fused(&plan) ); } @@ -16792,7 +16019,6 @@ async fn native_composed_binary_restores_the_context_after_a_refused_operand() { ctx: PromPlannerContext::from_eval_stmt(&stmt).unwrap(), promql_annotations: None, native_fused: true, - native_binary_offer: true, }; let before = format!("{:?}", planner.ctx); let refused = planner diff --git a/src/query/src/promql/planner/test/delta.rs b/src/query/src/promql/planner/test/delta.rs index 3913251bd8f..be0f5b3b28f 100644 --- a/src/query/src/promql/planner/test/delta.rs +++ b/src/query/src/promql/planner/test/delta.rs @@ -412,7 +412,6 @@ async fn delta_mixed_ranges_drop_and_float_ranges_sum() { ctx: PromPlannerContext::default(), promql_annotations: Some(collector.clone()), native_fused: false, - native_binary_offer: true, }; let left_context = direct_or_context("lhs", &["job", marker], "v"); let right_context = direct_or_context("rhs", &["job", marker], "v"); @@ -715,7 +714,6 @@ async fn binary_joins_align_only_the_temporality_marker() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let joined = planner .join_on_non_field_columns( @@ -756,7 +754,6 @@ async fn binary_joins_align_only_the_temporality_marker() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let set = planner .set_op_on_non_field_columns( @@ -813,7 +810,6 @@ async fn binary_joins_align_only_the_temporality_marker() { ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let set = planner .set_op_on_non_field_columns( @@ -927,7 +923,6 @@ async fn binary_joins_align_dictionary_temporality_marker_with_tagless_vector() ctx: PromPlannerContext::default(), promql_annotations: None, native_fused: false, - native_binary_offer: true, }; let scan = |name, table: Arc| { LogicalPlanBuilder::scan(name, provider_as_source(table), None) diff --git a/src/query/tests/native_instant_at.rs b/src/query/tests/native_instant_at.rs index 4f173120ff1..915b06c206c 100644 --- a/src/query/tests/native_instant_at.rs +++ b/src/query/tests/native_instant_at.rs @@ -340,27 +340,6 @@ fn native_anchored_timestamp_selection_boundaries() { fn timestamp_refuses_two_series_that_collapse_to_one_label_set() { let stmt = stmt("timestamp(a @ 1)", 0, 4000, 3000); let tree = compile(&stmt, schema(1), SeriesOrdering::KeyThenTime).unwrap(); - // The refusal is the canonical pass's, so this test makes the pass the planner makes: the facts - // it hands over for a scan bound to one metric are the labels that scan's series carry - - // `__name__` among them, named by the binding - and the one metric it is bound to. `timestamp` - // drops the metadata labels, and `__type__` is one of them, so two of that scan's series can - // become one label set; the pass therefore collects the pieces above the node, and the merge is - // what reports the collision. The node itself keeps no table to check. - let source_tags = vec![vec![ - "job".to_string(), - "__type__".to_string(), - "__name__".to_string(), - ]]; - let source_metrics = vec![Some(Arc::::from("a"))]; - // The scan's key columns always decode a value here, so the labels alone decide: `__type__` in - // the tags is what keeps this drop collected. - let tree = promql::native::canonicalize::merge_label_rewrites( - &tree, - &source_tags, - &source_metrics, - true, - ) - .expect("the canonical pass accepts the tree"); let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 24)); let state = NativeExecutionState::try_new(&tree, pool, None).unwrap(); let mut executor = NativeExecutor::new(state); diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index fadd81ae06a..ecb213c4ca3 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -209,7 +209,6 @@ macro_rules! http_tests { test_remote_write_v2_histogram_flush_native_consistency, test_remote_write_v2_histogram_native_partitions, test_remote_write_v2_histogram_native_range_function, - test_remote_write_v2_histogram_native_nullable_tag_boundary, test_prometheus_remote_write_batched, test_prometheus_remote_write_batched_mixed_time_index_units, test_prometheus_remote_write_batched_interceptor_time_index_units, @@ -1702,13 +1701,10 @@ fn assert_native_plan( ) -> Value { let fused = { let mut nodes = plan_nodes_named(plan, "FusedSeriesExec"); - // One node, or the pair a collected metadata drop is: the producer that runs the tree on - // each partition and the consumer that converts the assembled frames into rows. These - // fixture tables declare their tag column nullable, so the drop is collected and the pair is - // what a vector statement plans. - assert!( - (1..=2).contains(&nodes.len()), - "{context}: expected one fused node or the boundary's pair: {nodes:?}" + assert_eq!( + nodes.len(), + 1, + "{context}: expected one fused node: {nodes:?}" ); nodes.remove(0) }; @@ -2551,25 +2547,6 @@ pub async fn test_prometheus_native_promql_route(store_type: StorageType) { instance.guard.remove_all().await; } -/// The fused node `node` is, as its own line renders it. -fn describe_node(node: &Arc) -> String { - datafusion::physical_plan::displayable(node.as_ref()) - .one_line() - .to_string() -} - -/// The fused nodes of one request, as their own node lines render them, for a failure message. -fn describe_nodes(nodes: &[Arc]) -> Vec { - nodes - .iter() - .map(|node| { - datafusion::physical_plan::displayable(node.as_ref()) - .one_line() - .to_string() - }) - .collect() -} - /// Keeps the native fused node and the raw scan the request executed, for the figures only they /// record. /// @@ -2596,42 +2573,12 @@ impl NativeFusedObserver { assert_eq!( captured.len(), 1, - "`{context}`: the request executed {} fused nodes: {:?}", - captured.len(), - describe_nodes(&captured) + "`{context}`: the request executed {} fused nodes: {captured:?}", + captured.len() ); captured.remove(0) } - /// The fused node that ran the statement's own tree. - /// - /// A statement whose metadata drop is collected runs a canonical boundary: the producer runs the - /// tree on each scan partition and the consumer above the router only converts the assembled - /// frames into the statement's rows. Every figure below is read off the producer, because it is - /// the node that binds the source, reads the scan and reports the partitions it ran on; a - /// statement with nothing collected runs that node alone. - fn take_statement(&self, context: &str) -> Arc { - let mut captured = std::mem::take(&mut *self.0.lock().unwrap()); - assert!( - !captured.is_empty(), - "`{context}`: the request executed no fused node, so it left the native route" - ); - // At most the pair a boundary is: one producer and the consumer above it. - assert!( - captured.len() <= 2, - "`{context}`: the request executed {} fused nodes: {:?}", - captured.len(), - describe_nodes(&captured) - ); - if let Some(index) = captured - .iter() - .position(|node| promql::extension_plan::is_canonical_producer_exec(node.as_ref())) - { - return captured.remove(index); - } - captured.remove(0) - } - /// The one raw scan the request just executed. /// /// The fused node reads its region through a router over a merge scan, whose region half is @@ -2825,7 +2772,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { // The body is drained, so every partition has finished and the node this statement ran is // complete; its own registrations - not the flat set the JSON view reads - carry the // per-partition figures. - let exec = observer.take_statement(query); + let exec = observer.take(query); assert_native_plan(&plan, &exec, scan_partitions, query); assert_native_partition_output(&exec, scan_partitions, expected.len(), query); @@ -2884,12 +2831,10 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { "`{query}` did not execute the fused route" ); let fused_nodes = plan_nodes_named(&plan, "FusedSeriesExec"); - // The drop `* 100` makes over these fixture tables' nullable tag column is collected, so the - // native plan is that boundary: the producer that runs the tree and the consumer above it. An - // empty match changes which series are read, not which stages the statement is lowered into. - assert!( - (1..=2).contains(&fused_nodes.len()), - "`{query}`: expected one fused node or the boundary's pair: {fused_nodes:?}" + assert_eq!( + fused_nodes.len(), + 1, + "`{query}`: expected one fused node: {fused_nodes:?}" ); let scans = plan_nodes_named(&plan, "SeriesScanExec"); assert_eq!( @@ -2900,7 +2845,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { // The body is drained, so the one fused node the request ran has finished. Its own child list is // what says how many plans it reads, and its own line is where the tree it ran is still one text: // the JSON view lists that tree's nodes as children of the operator, next to the plans it reads. - let exec = observer.take_statement(query); + let exec = observer.take(query); assert_eq!( exec.children().len(), 1, @@ -5872,203 +5817,9 @@ pub async fn test_remote_write_v2_histogram_native_range_function(store_type: St body["data"]["result"][0]["histogram"][1]["sum"], "30", "`sum_over_time` adds the sums as {PROMETHEUS_REFERENCE} does" ); - // `sum_over_time` over a native histogram drops the metadata labels, and this table's tags are - // nullable, so the canonical pass collects that drop: the answer stands on a canonical merge - // carrying histogram frames. Lowering such a boundary is two nodes, not one - a producer that - // runs the statement's own tree per partition and exports a frame, and the consumer above the - // router that converts the merged frames into the statement's rows. Both are the fused route; - // the answer asserted above is what proves the histograms survived the crossing. - let fused = observer.take_all("histogram sum_over_time"); assert_eq!( - 2, - fused.len(), - "the canonical boundary is a producer and a consumer: {fused:?}" - ); - assert!( - fused.iter().all(|node| node.name() == "FusedSeriesExec"), - "the boundary executed a node that is not the fused producer: {fused:?}" - ); -} - -/// A native histogram answer crosses a canonical merge on a real metric table. -/// -/// Every tag column of a metric-engine table is nullable, so `keys_survive_metadata_drop` is false -/// on one and every statement whose kernel drops the metadata labels pays a canonical merge - the -/// `NULL`/`''` pair a drop normalizes into one label set is what the collection is for. For a -/// histogram answer that merge carries histogram frames, which is the pairing this test pins end to -/// end: the producer routes the pieces of one label set, the owner merges them with the same kernel -/// it merges floats with, and the consumer answers what it collected. -/// -/// The statement shapes here are the two sides of that crossing: `sum_over_time(h[1m])` answers the -/// histograms the merge collected, and `histogram_count(sum_over_time(h[1m]))` reduces them to -/// floats above it - so the boundary carries histograms while the statement answers a float. -pub async fn test_remote_write_v2_histogram_native_nullable_tag_boundary(store_type: StorageType) { - let observer = Arc::new(NativeFusedObserver::default()); - let plugins = Plugins::default(); - let wrapper: PhysicalPlanWrapperRef = observer.clone(); - plugins.insert(wrapper); - let instance = tests_integration::standalone::GreptimeDbStandaloneBuilder::new( - "remote_write_v2_histogram_native_nullable_tag_boundary", - ) - .with_default_store_type(store_type) - .with_query_options(QueryOptions { - experimental_native_promql: true, - ..Default::default() - }) - .with_plugin(plugins) - .build() - .await; - let server = build_test_prom_server(instance.fe_instance().clone(), false, None) - .with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap()) - .build(); - let client = TestClient::new(server.build(server.make_app()).unwrap()).await; - write_native_histogram( - &client, - &[ - (prom_store::METRIC_NAME_LABEL, "native_histogram_nullable"), - ("job", "api"), - ], - 1000, - 5, - 10.0, - ) - .await; - write_native_histogram( - &client, - &[ - (prom_store::METRIC_NAME_LABEL, "native_histogram_nullable"), - ("job", "api"), - ], - 3000, - 8, - 20.0, - ) - .await; - // The premise of this test, read off the table the writes created: the `job` tag is nullable, so - // a drop of the metadata labels is collected for the key columns rather than proved by them. - validate_data( - "native histogram nullable tag", - &client, - "select column_name, is_nullable from information_schema.columns \ - where table_schema = 'public' and table_name = 'native_histogram_nullable' \ - and column_name = 'job'", - "[[\"job\",\"YES\"]]", - ) - .await; - - // A histogram answer standing on the collected merge: `sum_over_time` over a native histogram - // drops the metadata labels, so the two writes' histograms travel through the boundary and the - // answer is the one histogram the merge collected. - observer.clear(); - let response = client - .get("/v1/prometheus/api/v1/query?query=sum_over_time(native_histogram_nullable%5B1m%5D)&time=3") - .header("x-greptime-hints", "query.experimental_native_promql=true") - .send() - .await; - assert_eq!(response.status(), StatusCode::OK); - let body: Value = response.json().await; - assert_eq!(body["data"]["resultType"], "vector", "{body}"); - assert_eq!(body["data"]["result"][0]["histogram"][0], 3.0, "{body}"); - assert_eq!( - body["data"]["result"][0]["histogram"][1]["count"], "13", - "`sum_over_time` adds the counts of the merged histograms: {body}" - ); - assert_eq!( - body["data"]["result"][0]["histogram"][1]["sum"], "30", - "`sum_over_time` adds the sums of the merged histograms: {body}" - ); - let fused = observer.take_all("histogram answer over a nullable tag"); - assert_eq!( - 2, - fused.len(), - "the collected drop is a producer and a consumer: {fused:?}" - ); - assert!( - fused.iter().all(|node| node.name() == "FusedSeriesExec"), - "the histogram boundary left the native route: {fused:?}" - ); - - // The same boundary with a float answer above it: `histogram_count` maps the histograms the - // merge collected to floats, so the frames crossing the boundary are still histograms while the - // statement publishes a float column. This is the shape the removed fallback used to refuse for - // the histogram case and the only one that ever worked here, and both now take one route. - observer.clear(); - let response = client - .get("/v1/prometheus/api/v1/query?query=histogram_count(sum_over_time(native_histogram_nullable%5B1m%5D))&time=3") - .header("x-greptime-hints", "query.experimental_native_promql=true") - .send() - .await; - assert_eq!(response.status(), StatusCode::OK); - let body: Value = response.json().await; - assert_eq!(body["data"]["resultType"], "vector", "{body}"); - assert_eq!( - body["data"]["result"][0]["value"][1].as_str(), - Some("13.0"), - "`histogram_count` counts the merged histograms' samples: {body}" - ); - let fused = observer.take_all("float answer over a histogram boundary"); - assert_eq!( - 2, - fused.len(), - "the collected drop is a producer and a consumer: {fused:?}" - ); - assert!( - fused.iter().all(|node| node.name() == "FusedSeriesExec"), - "the float answer left the native route: {fused:?}" - ); - - // `rate` is the same crossing over a window: a histogram answer whose count and sum the merge - // carries, compared against the existing plan's answer so the crossing is judged by the number - // rather than by the shape alone. - let url = "/v1/prometheus/api/v1/query?query=rate(native_histogram_nullable%5B5m%5D)&time=4"; - observer.clear(); - let native = client - .get(url) - .header("x-greptime-hints", "query.experimental_native_promql=true") - .send() - .await; - assert_eq!(native.status(), StatusCode::OK); - let native: Value = native.json().await; - let legacy = client.get(url).send().await; - assert_eq!(legacy.status(), StatusCode::OK); - let legacy: Value = legacy.json().await; - assert_eq!( - native["data"]["result"], legacy["data"]["result"], - "`rate(h[5m])`: the native answer differs from the existing plan's" - ); - // The two writes 2s apart are `count` 5 and `8` with `sum` 10 and 20, and the 5m window at the - // instant `4` extrapolates that over the range: (8 - 5) / 300s and (20 - 10) / 300s, in seconds. - assert_eq!( - ( - native["data"]["result"][0]["histogram"][1]["count"].as_str(), - native["data"]["result"][0]["histogram"][1]["sum"].as_str(), - ), - (Some("0.02"), Some("0.06666666666666667")), - "`rate(h[5m])` answered {native}" - ); - let fused = observer.take_all("histogram rate over a nullable tag"); - let descriptions = fused - .iter() - .map(|node| { - datafusion::physical_plan::displayable(node.as_ref()) - .one_line() - .to_string() - }) - .collect::>(); - assert!( - descriptions - .iter() - .all(|description| description.starts_with("FusedSeriesExec")), - "`rate(h[5m])` left the native route: {descriptions:?}" - ); - // How many fused nodes a raw window's boundary executes is a property of where its regions run - // it, so this asks what the executed plan carries instead: a canonical merge standing on the - // histogram window, which is the pairing the answer above is computed through. - assert!( - descriptions.iter().any(|description| description - .contains("Merge: canonical label-set merge") - && description.contains("Window: rate ")), - "`rate(h[5m])` executed no canonical merge over its histogram window: {descriptions:?}" + observer.take("histogram sum_over_time").name(), + "FusedSeriesExec" ); } @@ -6146,20 +5897,10 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag (expected_count.to_string(), expected_sum.to_string()), "{query}: {body}" ); - // Scalar arithmetic over a histogram drops the metadata labels, and this table's tags are - // nullable, so the answer stands on a canonical merge that carries histogram frames. The - // boundary is two fused nodes - the producer that scales each partition's histograms and the - // consumer above the router that answers the statement - and the scaled counts and sums - // asserted above are what proves the payload crossed it intact. - let fused = observer.take_all(query); assert_eq!( - 2, - fused.len(), - "{query}: the canonical boundary is a producer and a consumer: {fused:?}" - ); - assert!( - fused.iter().all(|node| node.name() == "FusedSeriesExec"), - "{query} left the native route: {fused:?}" + observer.take("histogram arithmetic").name(), + "FusedSeriesExec", + "{query} left the native route" ); } @@ -6178,18 +5919,12 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag Some(0), "`2 / h` should drop the histogram position: {body}" ); - // `2 / h` drops the histogram position and answers nothing, but it is still the same statement - // kind on the same collected drop, so it runs through the same two-node boundary: the producer - // scales each partition's histograms, drops them, and the consumer answers the empty result. - let fused = observer.take_all("histogram arithmetic: scalar over histogram"); assert_eq!( - 2, - fused.len(), - "`2 / h`: the canonical boundary is a producer and a consumer: {fused:?}" - ); - assert!( - fused.iter().all(|node| node.name() == "FusedSeriesExec"), - "`2 / h` left the native route: {fused:?}" + observer + .take("histogram arithmetic: scalar over histogram") + .name(), + "FusedSeriesExec", + "`2 / h` left the native route" ); } @@ -7548,25 +7283,16 @@ pub async fn test_remote_write_v2_histogram_sum_and_avg_native_route(store_type: &format!("{query}: the existing path answered something else: {legacy}"), ); - // Both halves of the split executed: a partial producer that folds its own input and a final - // consumer that merges the partials above the state router. The capture holds them once per - // stage that ran the split, so the count is not pinned; what is asserted is that every node - // is a fused stage of that split and that both halves are there - one alone would mean the - // other stage fell back to the existing plan. + // Both halves of the split executed: the partial producer folded each partition and the + // final consumer merged that state into the answer above. let fused = observer.take_all(query); - let described = describe_nodes(&fused); - assert!( - described - .iter() - .all(|node| node.contains(" partial") || node.contains(" final")), - "{query}: a node of the capture is not one half of the split: {described:?}" + assert_eq!( + fused.len(), + 2, + "{query}: expected the split's two nodes: {fused:?}" ); - for half in [" partial", " final"] { - assert!( - described.iter().any(|node| node.contains(half)), - "{query}: the split ran without its{half} half: {described:?}" - ); - } + let consumer = answer_consumer(&fused); + assert_eq!(consumer.name(), "FusedSeriesExec"); } } @@ -7647,26 +7373,28 @@ pub async fn test_remote_write_v2_histogram_avg_native_partitions(store_type: St "the partitioned average is wrong: {partitioned}" ); let fused = observer.take_all("avg over partitions"); + let consumer = answer_consumer(&fused); + assert_eq!(consumer.name(), "FusedSeriesExec"); + assert_eq!( + fused.len(), + 2, + "the split's partial and final nodes both execute: {fused:?}" + ); // The producer runs once per partition that read rows, which is what makes this a merge and not - // a single fold, so the figures this test reads are the partial stage's: it is the node whose - // tree carries the fold before the state router, and it is the only node that reports them. + // a single fold. let producer = fused .iter() - .find(|node| describe_node(node).contains(" partial")) - .expect("the split's partial producer"); - // The per-partition figures are the ones the node registered per partition, which is the view - // `node_metric_set` reports; the node's own aggregated set sums them into one entry. - let metrics = node_metric_set(producer); + .find(|node| !Arc::ptr_eq(node, consumer)) + .expect("the partial producer"); + let metrics = producer.metrics().expect("producer metrics"); let mut partitions: BTreeMap = BTreeMap::new(); for metric in metrics.iter() { - let Some(partition) = metric.partition() else { - continue; - }; - let MetricValue::Count { count, .. } = metric.value() else { - continue; - }; - if metric.value().name() == "native.root.output_values" { - *partitions.entry(partition).or_insert(0usize) += count.value(); + if metric.value().name() == "native.root.output_values" + && let MetricValue::Count { count, .. } = metric.value() + { + *partitions + .entry(metric.partition().expect("partition label")) + .or_insert(0usize) += count.value(); } } assert_eq!(