From 3630ba0488d7cdb32900f9d2fec6c37175f8f5fb Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:54:11 +0800 Subject: [PATCH] fix(promql): refuse an interleaved input, and close a label set only for its owner Two series-level identity holes, both reported against `42ea525`. The truncation flag followed the last chunk handed over, which is the same as "is a series open" only while at most one series can be open. A tree with no window still accepted `Interleaved`, and `a` open, `b` closed, end of input then passed while `a` never closed. Tracking the open set would need an identity that a raw chunk in bound-label mode does not have (`metric` is `None`, the key holds it), so the ordering is refused when the state is built instead, for every tree shape and not only under a window. The flag's doc now says what it means and why that is the whole answer. `History` closed on any series' `is_last`, so with `abs` dropping the metric name a series that contributed nothing could close a label set another series was still filling: `a{job="x"}` value, `b{job="x"}` empty close, `a{job="x"}` continuation was refused with "already closed the label set". Contiguous input hands over exactly that sequence, so it was not an interleaving artefact. A history now remembers the series that filled it and closes only for that series' closing chunk; a series that produced nothing under an open set is passed over, as it already was under a closed one. The owner is charged as part of the entry. The ordering check also reads every input node rather than only the first: a tree with two inputs would otherwise have its second declaration read and dropped. Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --- src/promql/src/native/execute.rs | 226 +++++++++++++++++-------- src/promql/src/native/execute/tests.rs | 187 +++++++++++++++----- 2 files changed, 297 insertions(+), 116 deletions(-) diff --git a/src/promql/src/native/execute.rs b/src/promql/src/native/execute.rs index 4f9853b6b2d..b1e2afe9a28 100644 --- a/src/promql/src/native/execute.rs +++ b/src/promql/src/native/execute.rs @@ -51,13 +51,13 @@ use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReser use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datatypes::arrow::array::{Array, Float64Array}; use datatypes::arrow::buffer::Buffer; -use datatypes::series::SeriesOrdering; use datatypes::series::histogram::HistogramBatch; use datatypes::series::raw::RawSeriesChunk; use datatypes::series::samples::{ DenseSamplesRef, SampleRef, Samples, Series, SeriesFields, SeriesSchema, }; use datatypes::series::time::{GridRange, TimeAxis}; +use datatypes::series::{MetricName, SeriesOrdering}; use crate::grid_series::{LabelSet, label_set}; use crate::map::{MathOp, drop_metric_name, map_math}; @@ -99,6 +99,17 @@ impl NativeChunk { } } + /// The identity of the series this chunk belongs to, when the chunk names it. + /// + /// A raw chunk in bound-label mode carries its identity in its key and no metric name, so there + /// is nothing to compare; an evaluated chunk always names the series it holds. + fn metric(&self) -> Option<&MetricName> { + match self { + Self::Raw(chunk) => chunk.metric.as_ref(), + Self::Evaluated(chunk) => Some(&chunk.series.metric), + } + } + /// The bytes this chunk holds, counting each shared allocation it references that `charged` has /// not met yet. fn bytes(&self, charged: &mut ChargedAllocations) -> usize { @@ -528,13 +539,37 @@ struct Emitted { struct History { /// The occupied runs of each field, in position order and disjoint. fields: Vec>, + /// The source series that filled this label set, when the chunk that did named it. + /// + /// A chunk holding no value is handed over to close a series, never to fill a label set, so it + /// closes this one only when it belongs to the series that filled it: a second series that + /// produced nothing here is a series this label set never saw, and closing on its `is_last` + /// would leave the series that did produce the values unable to finish. `None` is a set filled + /// by chunks that named no series, where there is nothing to compare a closing chunk against. + owner: Option, /// Whether the series on this label set closed. A second series cannot close it again. closed: bool, - /// The heap this entry owns: the runs its per-field vectors hold, kept current by - /// [`Emitted::occupy`]. The header and the slot it sits in are charged by the table's capacity. + /// The heap this entry owns: the runs its per-field vectors hold and the owner's name and tags, + /// kept current by [`Emitted::occupy`]. The header and the slot it sits in are charged by the + /// table's capacity. bytes: usize, } +impl History { + /// Whether a chunk of `source` holding no value is the one that closes this label set. + /// + /// Only the series that filled it can: a series that produced nothing here is one this label + /// set never saw, so closing on its `is_last` would take the close away from the series that + /// did fill it. A chunk naming no series is compared with nothing and closes the set, which is + /// what every chunk did before the owner was kept. + fn closed_by(&self, source: Option<&MetricName>) -> bool { + match (self.owner.as_ref(), source) { + (Some(owner), Some(source)) => owner == source, + _ => true, + } + } +} + /// The bytes a label set holds: each label's name and value with the pair that refers to them. fn label_bytes(labels: &LabelSet) -> usize { labels @@ -543,7 +578,23 @@ fn label_bytes(labels: &LabelSet) -> usize { .sum() } -/// The bytes a history holds: the per-field vectors and the runs in them. +/// The bytes a metric name holds: its name and each tag's name and value, with the tags that refer +/// to them. +/// +/// This is the convention [`label_bytes`] keeps: the payload of the shared strings and one row per +/// label. The `Option` a [`History`] keeps its owner in is not counted here, being part +/// of the header that entry's slot is charged for. +fn metric_bytes(metric: &MetricName) -> usize { + metric.name().map_or(0, |name| name.len()) + + std::mem::size_of_val(metric.tags()) + + metric + .tags() + .iter() + .map(|tag| tag.name.len() + tag.value.len()) + .sum::() +} + +/// The bytes a history holds: the per-field vectors, the runs in them and its owner. /// /// This is what the entry itself allocates, not the slot it is filed under: the `History` lives in /// the map's slot, and that slot is what [`Emitted::bytes`] charges with the table's capacity. A @@ -556,11 +607,16 @@ fn history_bytes(history: &History) -> usize { .iter() .map(|runs| runs.capacity() * std::mem::size_of::()) .sum::() + + history.owner.as_ref().map_or(0, metric_bytes) } impl Emitted { /// Records what `chunk` occupies, rejecting it against what this node already handed out. /// + /// `source` is the identity of the input series `chunk` came from, when the chunk names one: a + /// label set is filled by the series that contributes its values, and it is that series whose + /// closing chunk closes it (see [`History::owner`]). + /// /// A label set that has closed cannot be reached by a second series: the node would hand on two /// chunks that its consumer cannot tell apart, and the merge the caller wants spans the first /// series, whose state the evaluator above has already closed. Such a pair does merge in @@ -568,7 +624,12 @@ impl Emitted { /// refused instead of being answered wrongly (see [`closed_label_set`]). A chunk of a set still /// open is refused only where it holds a value at a position that same field already holds one /// at, which is the case no merge can resolve. - fn occupy(&mut self, path: &NodePath, chunk: &EvaluatedChunk) -> Result<()> { + fn occupy( + &mut self, + path: &NodePath, + source: Option<&MetricName>, + chunk: &EvaluatedChunk, + ) -> Result<()> { let labels = label_set(&chunk.series.metric); let occupied: Vec> = chunk .series @@ -577,18 +638,18 @@ impl Emitted { .enumerate() .map(|(index, field)| occupied_runs(path, index, field, &chunk.series.time)) .collect::>()?; - // A chunk holding no value records nothing. If the label set is unknown or already closed, - // the series this chunk belongs to contributed no sample at all -- either it is the first - // such series, or a later one that also answers empty -- so there is nothing to report, - // nothing to merge and nothing to refuse. A label set still open is the other case: the - // chunk closes the series that *did* produce the values, so it falls through to close it. let empty = occupied.iter().all(Vec::is_empty); - if empty && self.sets.get(&labels).is_none_or(|history| history.closed) { - return Ok(()); - } - if let Some(history) = self.sets.get(&labels) { + let recorded = self.sets.get(&labels); + if let Some(history) = recorded { if history.closed { - return Err(closed_label_set(path, &labels)); + // A series that produced nothing under a closed label set contributes no sample, so + // there is nothing to report, nothing to merge and nothing to refuse. One that did + // produce values is the second series this set cannot take. + return if empty { + Ok(()) + } else { + Err(closed_label_set(path, &labels)) + }; } for (index, runs) in occupied.iter().enumerate() { let stored = history.fields.get(index).map_or(&[][..], Vec::as_slice); @@ -604,6 +665,17 @@ impl Emitted { } } } + // A chunk holding no value records nothing. Past a closed set the same series is the + // one that answered empty, and on an open set it is the close of the series that filled + // it -- but only of that series: a different one closes a label set it never wrote to, + // which would leave the series that did write to it unable to finish. + if empty && !(chunk.is_last && history.closed_by(source)) { + return Ok(()); + } + } else if empty { + // The first chunk onto a label set holds no value: a series that answered empty, with + // nothing to record, nothing to close and nothing to merge. + return Ok(()); } let fields = chunk.series.fields.len(); // The change is folded in here rather than recomputed when the figure is read: a node @@ -617,6 +689,14 @@ impl Emitted { } }; let before = history.bytes; + // The series that contributes the values is the one that owns the label set, so a chunk of + // another one cannot close it later on. + if !empty + && let Some(source) = source + && history.owner.as_ref() != Some(source) + { + history.owner = Some(source.clone()); + } if history.fields.len() < fields { history.fields.resize_with(fields, Vec::new); } @@ -638,10 +718,11 @@ impl Emitted { /// An estimate of the bytes this table holds, which the node reserves for. /// /// It counts each label's name and value bytes with a tuple per label, the runs a history keeps - /// in its per-field vectors, and the table's capacity at one label-set slot plus one history - /// slot per entry -- which is where a `History` header itself is charged, once, for the entry - /// that lives in it. It leaves out the map's buckets and the `Arc` headers of the strings it - /// shares, so it is a lower bound rather than a measurement. + /// in its per-field vectors, the owner each entry keeps with its name and tags, and the table's + /// capacity at one label-set slot plus one history slot per entry -- which is where a `History` + /// header itself is charged, once, for the entry that lives in it. It leaves out the map's + /// buckets and the `Arc` headers of the strings it shares, so it is a lower bound rather than a + /// measurement. /// /// The per-entry part is a running total, so reading this after every output does not walk the /// table: only the entry that changed and the table's own capacity are read here. @@ -950,12 +1031,10 @@ struct NodeRuntime { /// consumer saw did not close its series, and no chunk came after it. Nothing else writes it, so /// it cannot report a truncation that did not happen. /// - /// One flag is enough under the `Contiguous` and `KeyThenTime` orderings, which keep the chunks - /// of a series together: only the series last handed over can be open, and one that closed - /// cannot resume. An `Interleaved` input does reach a tree with no window above it -- that - /// ordering is refused only where a window reads a series at a time -- and there an earlier - /// series left open behind another one's chunk is not reported. The flag under-reports such a - /// stream; it never over-reports one. + /// "The series of the last chunk handed over" is the whole answer because no more than one + /// series can be open here: [`NativeExecutionState::try_new`] refuses an input declaring + /// `SeriesOrdering::Interleaved`, so the chunks of a series arrive together and the series last + /// handed over is the only one that could still be open. open_source_series: bool, } @@ -1060,8 +1139,10 @@ impl NodeRuntime { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - // What it handed out is what the label sets of this node now hold. - self.track(path, output)?; + // What it handed out is what the label sets of this node now hold. The identity is the + // input's: `map_chunk` has already rewritten the output's label set, and a series is + // the one that arrived. + self.track(path, Some(&chunk.series.metric), output)?; } let bytes = self.emitted_bytes(); self.buffer.hold_exactly(bytes)?; @@ -1075,6 +1156,12 @@ impl NodeRuntime { chunk.positions(), chunk.is_last(), ); + // Which series this chunk came from, for the label-set table below. A window consumes one + // series to its `is_last` before the next, so the chunk that closes an output label set is + // the one of the series that produced its values, which is what lets the table accept a + // closing chunk of a series that wrote to it. A raw chunk in bound-label mode names no + // series, so there is nothing to pass on. + let source = chunk.metric(); // The chunk is held for as long as the evaluator consumes it. self.buffer.hold(bytes)?; let metrics = Arc::clone(&self.buffer.metrics); @@ -1098,8 +1185,9 @@ impl NodeRuntime { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - // What it handed out is what the label sets of this node now hold. - self.track(path, output)?; + // What it handed out is what the label sets of this node now hold, and the series that + // filled one is the series its chunks came from. + self.track(path, source, output)?; } // The chunk has been consumed: what this node still holds is what the evaluator kept of it // and the label sets it has already handed out. Between pushes the charge for both is the @@ -1131,13 +1219,20 @@ impl NodeRuntime { /// Records what this node hands out under its output label set. /// - /// A node that rewrites labels is where two input series become indistinguishable, so this is - /// where the second one is refused: the node above reads what this one hands over, and a window - /// above would evaluate the two as two series and report both. Checking at the root would be - /// too late for that — the answer would already have been computed. - fn track(&mut self, path: &NodePath, output: &EvaluatedChunk) -> Result<()> { + /// `source` is the identity of the input series the chunk came from, when there is one: a node + /// that rewrites labels is where two input series become indistinguishable, so this is where + /// the second one is refused, and it is also the only place that knows which series filled the + /// label set a chunk closes. A node above reads what this one hands over, and a window above + /// would evaluate the two as two series and report both. Checking at the root would be too late + /// for that — the answer would already have been computed. + fn track( + &mut self, + path: &NodePath, + source: Option<&MetricName>, + output: &EvaluatedChunk, + ) -> Result<()> { match self.emitted.as_mut() { - Some(emitted) => emitted.occupy(path, output), + Some(emitted) => emitted.occupy(path, source, output), // A node that keeps its input's labels cannot make two series one. None => Ok(()), } @@ -1209,23 +1304,23 @@ impl NativeExecutionState { // cannot be shown at one position and registered at another. let mut pending: Vec = Vec::new(); let mut unsupported: Option = None; - // The order the source streams its series in, and where it says so: a window evaluates one - // series at a time, so it is only executable over an input that keeps each series - // contiguous. - let mut ordering: Option<(SeriesOrdering, NodePath)> = None; - let mut first_window: Option = None; + // The first input whose declared order this runtime cannot execute, and where it says so. + // Every node here reads one series at a time, so an input that interleaves them is one this + // runtime cannot execute. Recorded per input rather than kept from the first one seen: a + // tree with two inputs would otherwise have its second declaration read and dropped. + let mut unsupported_ordering_found: Option<(SeriesOrdering, NodePath)> = None; metrics::walk(tree.root(), &mut |node, path| { let handles = Arc::new(NodeMetrics::default()); handles.register(path, metrics_set, partition); let plan = match &node.expr { NativeExpr::Input(bound) => { - ordering.get_or_insert_with(|| (bound.ordering, path.clone())); + if bound.ordering == SeriesOrdering::Interleaved { + unsupported_ordering_found + .get_or_insert_with(|| (bound.ordering, path.clone())); + } Some((NodeTag::Input, Seed::Input)) } - NativeExpr::Window { op, .. } => { - first_window.get_or_insert_with(|| path.clone()); - Some((NodeTag::Window, Seed::Window(op.clone()))) - } + NativeExpr::Window { op, .. } => Some((NodeTag::Window, Seed::Window(op.clone()))), NativeExpr::Map { op, .. } => Some((NodeTag::Map, Seed::Map(Arc::clone(&op.ops)))), // The kinds below are legal in a bound tree, and `validate` stays the authority on // that; this runtime simply does not execute them yet. @@ -1262,19 +1357,16 @@ impl NativeExecutionState { return Err(error); } - // A window feeds one series at a time to its evaluator, so it can only run over an input - // that keeps each series contiguous. The tree stays free to say `Interleaved` -- the - // compiler's job is not narrowed here -- but this runtime says so up front instead of - // failing on the second series of a chunk stream it advertised a chunk at a time. - if let Some(window) = &first_window - && let Some((ordering, source)) = &ordering - { - match ordering { - SeriesOrdering::Contiguous | SeriesOrdering::KeyThenTime => {} - SeriesOrdering::Interleaved => { - return Err(unsupported_ordering(*ordering, window, source)); - } - } + // Every node here reads one series at a time and assumes its chunks arrive together: a + // window feeds a series to its evaluator up to the chunk carrying `is_last`, and an input + // node keeps one flag for the series it handed over, which is what reports one that never + // closed. An input that declares `Interleaved` can leave a series open across another + // series' chunks, which either of those would misread, so the tree is refused here rather + // than run. That is a precondition of the runtime, not a limitation to document: the tree + // stays free to declare the ordering -- the compiler's job is not narrowed here -- and no + // tree this runtime executes declares it. + if let Some((ordering, source)) = &unsupported_ordering_found { + return Err(unsupported_ordering(*ordering, source)); } // A node's input is the node at the same position extended by the step the traversal took, @@ -1804,19 +1896,15 @@ fn closed(path: &NodePath) -> DataFusionError { DataFusionError::Execution(format!("the window at {path} is already closed")) } -/// The rejection of an input ordering this runtime cannot execute under a window. +/// The rejection of an input ordering this runtime cannot execute. /// -/// It names the window that needs contiguity, the input that does not keep it, and the ordering -/// that was found, so a reader can tell a tree this runtime cannot run from one it cannot run -/// correctly. -fn unsupported_ordering( - ordering: SeriesOrdering, - window: &NodePath, - input: &NodePath, -) -> DataFusionError { +/// It names the input that declares the ordering and the ordering found, so a reader can tell a +/// tree this runtime cannot run from one it cannot run correctly. +fn unsupported_ordering(ordering: SeriesOrdering, input: &NodePath) -> DataFusionError { DataFusionError::NotImplemented(format!( - "the native runtime does not execute windows over {ordering:?} input yet: the window at \ - {window} reads one series at a time, but the input at {input} is {ordering:?}" + "the native runtime does not execute {ordering:?} input yet: the nodes here read one series \ + at a time, and the input at {input} is {ordering:?}, so the chunks of two of them can \ + arrive interleaved" )) } diff --git a/src/promql/src/native/execute/tests.rs b/src/promql/src/native/execute/tests.rs index ddb96a57444..93f810f83e7 100644 --- a/src/promql/src/native/execute/tests.rs +++ b/src/promql/src/native/execute/tests.rs @@ -1349,9 +1349,13 @@ fn a_merge_across_a_closed_series_is_not_implemented() { /// Two open chunks that share a label set are accepted where no position of a field collides. /// /// `a{job="x"}` and `b{job="x"}` are two series to the input and one label set to `abs`, which -/// drops the name. The source is interleaved, so the second one is pushed while the first is still -/// open, and neither says it is the last chunk of its series. The two occupy disjoint positions of -/// field 0 — a value at 0 and a value at 3 — so the label-set check hands both on. +/// drops the name. Neither piece says it is the last chunk of its series, so the second one arrives +/// while the first is still open. The two occupy disjoint positions of field 0 — a value at 0 and a +/// value at 3 — so the label-set check hands both on. +/// +/// The ordering is contiguous because that is the only ordering this runtime executes: the stream +/// below is not one a scan produces, since it never closes `a`, and what is checked here is the +/// occupancy the check reads rather than a delivery order. /// /// This is acceptance by that check, and it is not a merge: this layer passes every chunk through /// as it arrives and keeps no sample state that would combine two series. Merging two series under @@ -1361,10 +1365,10 @@ fn a_merge_across_a_closed_series_is_not_implemented() { fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides() { let map_grid = grid(0, 60_000, 4); let tree = NativeTree::try_new(map( - evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Interleaved), + evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Contiguous), vec![MathOp::Abs], )) - .expect("a map-only tree is legal over an interleaved input"); + .expect("a map-only tree is legal over a contiguous input"); let schema = evaluated_schema(Arc::clone(&map_grid)); let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); @@ -1645,6 +1649,12 @@ fn a_chunk_with_no_output_keeps_charging_the_emitted_table() { } /// A window cannot run over an interleaved input, and the runtime says so before reading a chunk. +/// +/// The binding refuses a window whose own input declares the ordering, and the runtime refuses the +/// ordering itself -- which is what catches a window over a map over the source, where the binding +/// only sees the map. A tree with no window above the source is refused by the same check, since +/// every node here reads one series at a time; [`an_interleaved_input_is_refused_without_a_window`] +/// is that case on its own. #[test] fn a_window_over_an_interleaved_input_is_rejected() { let inner_grid = grid(0, 60_000, 2); @@ -1688,44 +1698,37 @@ fn a_window_over_an_interleaved_input_is_rejected() { ); let message = error.to_string(); assert!(message.contains("Interleaved"), "{message}"); - assert!(message.contains("at root "), "{message}"); assert!(message.contains("root.input"), "{message}"); +} - // A map needs no ordering: it holds one chunk at a time, so the series of an interleaved stream - // never have to be contiguous. `A1, B1, A2, B2` runs here as it arrives. - let map_grid = grid(0, 60_000, 4); +/// An interleaved input is refused with no window above it to notice it. +/// +/// A map over an input has no evaluator, so nothing in the tree reads one series to its `is_last`. +/// The sequence the input node cannot tell from a whole one is `a{job="x"}` open, `b{job="y"}` +/// closed, and the end of the input: the flag that reports a series left open would say the last +/// chunk closed its series and `finish` would accept an input whose first series never closed. The +/// runtime therefore refuses the ordering when the state is built, before a chunk is read, instead +/// of reporting a truncation it cannot see. +#[test] +fn an_interleaved_input_is_refused_without_a_window() { + let map_grid = grid(0, 60_000, 2); let tree = NativeTree::try_new(map( evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Interleaved), vec![MathOp::Abs], )) - .expect("a map-only tree is legal over an interleaved input"); - let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - for (name, job, start, values, is_last) in [ - ("a", "x", 0, vec![Some(-1.0), Some(-2.0)], false), - ("b", "y", 0, vec![Some(-3.0), Some(-4.0)], false), - ("a", "x", 2, vec![Some(-5.0), Some(-6.0)], true), - ("b", "y", 2, vec![Some(-7.0), Some(-8.0)], true), - ] { - execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric(name, job), - start, - values, - is_last, - )) - .unwrap(); - } - assert_eq!( - addressed(&run(&mut execution)), - vec![ - (0, vec![Some(1.0), Some(2.0)]), - (0, vec![Some(3.0), Some(4.0)]), - (2, vec![Some(5.0), Some(6.0)]), - (2, vec![Some(7.0), Some(8.0)]), - ] + .expect("the binding accepts a map over an interleaved input"); + let error = + match NativeExecutionState::try_new(&tree, pool(), &ExecutionPlanMetricsSet::new(), 0) { + Ok(_) => panic!("the runtime accepted an interleaved input with no window above it"), + Err(error) => error, + }; + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" ); + let message = error.to_string(); + assert!(message.contains("Interleaved"), "{message}"); + assert!(message.contains("root.input"), "{message}"); } /// The charge for the label sets a node has handed out does not depend on how it is read. @@ -1781,6 +1784,11 @@ fn the_emitted_charge_is_the_same_folded_in_as_walked() { /// This is the other order of [`two_series_with_no_result_do_not_collide`], and the one that used to /// fail: by the time the empty series arrives the label set is already closed, and refusing it /// failed an execution whose answer was never in doubt. +/// +/// The set is closed here, which is what makes the empty series a later one that also answers +/// empty. Its mirror image -- an empty closing chunk that arrives while the set is still open -- is +/// [`an_empty_closing_chunk_of_another_series_does_not_close_a_label_set`], where the same series +/// is not a harmless extra but would take the close away from the series that filled the set. #[test] fn a_series_with_no_result_after_a_closed_one_is_passed_over() { let map_grid = grid(0, 60_000, 1); @@ -1823,6 +1831,68 @@ fn a_series_with_no_result_after_a_closed_one_is_passed_over() { ); } +/// An empty closing chunk of one series does not close a label set another series is still filling. +/// +/// `abs` drops the metric name, so `a{job="x"}` and `b{job="x"}` are one label set to the map. `b` +/// holds no row at all: its closing chunk closes a series that contributed nothing, so it must not +/// close the set `a` opened, or `a`'s continuation would be refused for a label set that `a` itself +/// never closed. Two series back to back are what contiguous input hands over, so this is a stream +/// the runtime executes rather than an ordering it refuses. +#[test] +fn an_empty_closing_chunk_of_another_series_does_not_close_a_label_set() { + let map_grid = grid(0, 60_000, 2); + let tree = NativeTree::try_new(map( + evaluated_input(Arc::clone(&map_grid)), + vec![MathOp::Abs], + )) + .unwrap(); + let schema = evaluated_schema(Arc::clone(&map_grid)); + let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + + // `a` opens the label set and has not closed it when `b` closes an empty series onto it, and the + // chunk that closes `a` itself comes last. + let chunks = [ + evaluated_chunk_of(&schema, named_metric("a", "x"), 0, vec![Some(-1.0)], false), + evaluated_chunk_of(&schema, named_metric("b", "x"), 0, vec![], true), + evaluated_chunk_of(&schema, named_metric("a", "x"), 1, vec![Some(-2.0)], true), + ]; + let steps = chunks.len(); + let mut handed = Vec::new(); + for (step, chunk) in chunks.into_iter().enumerate() { + execution.push_evaluated(chunk).unwrap(); + while let Some(chunk) = execution + .next_chunk() + .unwrap_or_else(|error| panic!("step {step} was refused: {error}")) + { + handed.push(chunk); + } + } + assert_eq!( + handed.len(), + steps, + "the map did not consume all {steps} chunks it was given: {:?}", + decided(&handed) + ); + let last = handed.last().expect("the map handed nothing on"); + assert!(last.is_last, "the final output does not close its series"); + // `abs` drops the metric name, so the label set the output closes is `{job="x"}`. + let mut mapped = named_metric("a", "x"); + mapped.remove("__name__"); + assert_eq!( + last.series.metric, mapped, + "the final output closes a label set other than the one `abs` leaves of `a{{job=\"x\"}}`" + ); + assert_eq!( + (series_start(last), sample_values(last)), + (1, vec![Some(2.0)]), + "the value `a` held at position 1 did not survive" + ); + assert!( + execution.finish().unwrap().is_empty(), + "the map still held a chunk when the input ended" + ); +} + /// Two series that each produce no value at all are not a collision: both answer empty, and the /// node hands nothing on rather than refusing the second one. /// @@ -1958,10 +2028,10 @@ fn a_sparse_chunk_is_read_at_the_positions_it_holds_through_window_map_window() fn a_sparse_chunk_occupies_only_the_positions_it_holds() { let map_grid = grid(0, 60_000, 8); let tree = NativeTree::try_new(map( - evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Interleaved), + evaluated_input(Arc::clone(&map_grid)), vec![MathOp::Abs], )) - .expect("a map-only tree is legal over an interleaved input"); + .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); @@ -2527,7 +2597,7 @@ fn a_series_whose_axis_and_fields_disagree_is_rejected() { let mut emitted = Emitted::default(); let error = emitted - .occupy(&NodePath::root(), &chunk) + .occupy(&NodePath::root(), Some(&chunk.series.metric), &chunk) .expect_err("a field with fewer samples than the axis places rows was accepted"); let message = error.to_string(); assert!( @@ -2543,8 +2613,8 @@ fn a_series_whose_axis_and_fields_disagree_is_rejected() { /// The `History` of a label set lives in the map's slot, so a `size_of::()` term in the /// entry's own figure billed one header twice: once here and once in the capacity term beside it. /// The two assertions below are the definition of the whole figure -- the labels, the vectors and -/// their runs, and one label-set slot plus one history slot per entry -- so either header counted -/// in the wrong place fails them. +/// their runs, the owner's name and tags, and one label-set slot plus one history slot per entry -- +/// so either header counted in the wrong place fails them. #[test] fn a_history_entry_is_charged_once_for_its_header() { let grid = grid(0, 10_000, 2); @@ -2553,15 +2623,32 @@ fn a_history_entry_is_charged_once_for_its_header() { let labels = label_set(&metric); // Not the closing chunk: a closed label set drops its runs, and the runs are what the entry's // own figure is. - let chunk = evaluated_chunk_of(&schema, metric, 0, vec![Some(1.0), Some(2.0)], false); + let chunk = evaluated_chunk_of( + &schema, + metric.clone(), + 0, + vec![Some(1.0), Some(2.0)], + false, + ); let mut emitted = Emitted::default(); - emitted.occupy(&NodePath::root(), &chunk).unwrap(); + emitted + .occupy(&NodePath::root(), Some(&metric), &chunk) + .unwrap(); let history = emitted .sets .get(&labels) .expect("the chunk recorded its label set"); + // The owner the entry keeps is the series that filled it, so its name and each tag cost the + // same bytes the label set of the chunk did. + let owner = metric.name().map_or(0, |name| name.len()) + + std::mem::size_of_val(metric.tags()) + + metric + .tags() + .iter() + .map(|tag| tag.name.len() + tag.value.len()) + .sum::(); let entry = label_bytes(&labels) .saturating_add(history.fields.capacity() * std::mem::size_of::>()) .saturating_add( @@ -2570,7 +2657,13 @@ fn a_history_entry_is_charged_once_for_its_header() { .iter() .map(|runs| runs.capacity() * std::mem::size_of::()) .sum::(), - ); + ) + .saturating_add(owner); + assert_eq!( + history.owner.as_ref(), + Some(&metric), + "the entry does not name the series that filled it" + ); assert!( history.fields.iter().any(|runs| !runs.is_empty()), "the entry kept no run, so it has no heap of its own to charge for" @@ -2578,8 +2671,8 @@ fn a_history_entry_is_charged_once_for_its_header() { assert_eq!( emitted.entries, entry, - "the entry's own bytes are not its labels and runs alone: a `History` header in the figure \ - would add the {} bytes its slot already charges", + "the entry's own bytes are not its labels, its runs and its owner: a `History` header in \ + the figure would add the {} bytes its slot already charges", std::mem::size_of::(), ); assert_eq!( @@ -2625,7 +2718,7 @@ fn a_sparse_series_that_moves_forward_is_appended_to_its_history() { false, ); emitted - .occupy(&path, &chunk) + .occupy(&path, Some(&chunk.series.metric), &chunk) .unwrap_or_else(|error| panic!("position {position} was refused: {error}")); }; // The runs stored under the one label set, which is what `absorb` keeps in position order.