From da9c63dc0e15835e28418e5470101cb371222ea0 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:44:32 +0800 Subject: [PATCH] refactor(promql): remove NodePath and own the runtime tree recursively A node was identified by the path that reached it, so every execution and every metric export walked that path again, and a node's own state lived in tables keyed by it. This replaces that with one recursive runtime tree: each node owns its children, its operator state, its metrics and, once settled, nothing it no longer needs. An `InputId` is assigned only to the leaves that actually receive a feed, by a depth-first walk of the built tree, so a scalar leaf takes no id and no node is addressed by position. Pulling, settling, flushing after a subtree closes, EOF, errors and drop all recurse through the same tree. Signed-off-by: discord9 --- src/promql/src/extension_plan/fused_series.rs | 69 +- .../src/extension_plan/fused_series/tests.rs | 53 +- src/promql/src/native/aggregate.rs | 25 +- src/promql/src/native/display.rs | 63 +- src/promql/src/native/execute.rs | 2098 +++++++---------- src/promql/src/native/execute/tests.rs | 1964 +++++++-------- src/promql/src/native/metrics.rs | 448 +--- src/promql/tests/native_binary.rs | 47 +- src/query/src/promql/native_compile/tests.rs | 22 +- 9 files changed, 2099 insertions(+), 2690 deletions(-) diff --git a/src/promql/src/extension_plan/fused_series.rs b/src/promql/src/extension_plan/fused_series.rs index d957851902a..1d965dd3c1c 100644 --- a/src/promql/src/extension_plan/fused_series.rs +++ b/src/promql/src/extension_plan/fused_series.rs @@ -61,7 +61,7 @@ use datafusion::execution::memory_pool::{ }; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; -use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements, @@ -76,8 +76,10 @@ use store_api::region_engine::{RawLabelSource, SendableRawSeriesStream}; use crate::evaluated::EvaluatedChunk; use crate::grid_series::{LabelSet, label_set}; +use crate::native::execute::InputId; +use crate::native::execute::RuntimeTreeHandle; use crate::native::execute::{NativeExecutionState, NativeExecutor}; -use crate::native::metrics::{self, NodePath}; +use crate::native::metrics; use crate::native::plan::{NativeExpr, NativeTree}; use crate::native::runs::{Run, absorb}; use crate::source::materialize_raw_labels; @@ -404,9 +406,9 @@ impl PartialOrd for FusedSeries { /// can only agree or disagree with it, and the frame this node reads has to be decoded against the /// same schema, so both halves read it here instead of each carrying a copy. struct TreeInput { - /// The position the tree's raw chunks enter at: a position is how the runtime addresses a node, - /// so nothing on the node has to carry an identity that the tree itself does not. - path: NodePath, + /// Which of the tree's inputs the raw chunks enter at. A tree this node feeds has one leaf, so + /// this is the first input; it is kept here so the exec does not have to count inputs again. + input_id: InputId, /// The raw series schema the tree's input node was compiled against. schema: Arc, } @@ -414,17 +416,17 @@ struct TreeInput { /// Reads what the tree's input node was compiled against, rejecting a tree this node cannot feed. /// /// A node feeds one raw input, and its output is addressed on a grid; a tree that does not have -/// that shape is one this layer cannot hand chunks to or read positions from. +/// that shape is one this layer cannot hand chunks to. Because exactly one input is accepted, that +/// input is the tree's only leaf and is therefore the first one: the id it is queued at is +/// [`InputId::new(0)`], so nothing here has to count leaves beyond checking that it is alone. fn tree_input(tree: &NativeTree) -> DataFusionResult { let mut sources = 0; - let mut input_path = None; let mut schema = None; - metrics::walk(tree.root(), &mut |node, path| { + metrics::walk(tree.root(), &mut |node, _| { let NativeExpr::Input(input) = &node.expr else { return; }; sources += 1; - input_path.get_or_insert_with(|| path.clone()); // Raw, not evaluated: this node feeds the tree from a scan, so a tree whose input was // compiled against an already-evaluated schema would read the wrong thing. schema = input @@ -438,7 +440,7 @@ fn tree_input(tree: &NativeTree) -> DataFusionResult { "FusedSeries runs a tree with {sources} inputs, and it needs exactly one" ))); } - let (Some(path), Some(schema)) = (input_path, schema) else { + let Some(schema) = schema else { return Err(DataFusionError::Plan( "FusedSeries runs a tree whose input is not raw series".to_string(), )); @@ -453,7 +455,10 @@ fn tree_input(tree: &NativeTree) -> DataFusionResult { "FusedSeries runs a tree whose output is not evaluated on a grid".to_string(), )); } - Ok(TreeInput { path, schema }) + Ok(TreeInput { + input_id: InputId::new(0), + schema, + }) } /// Refuses a tree whose logical series can be split across the partitions this node runs. @@ -749,8 +754,8 @@ pub struct FusedSeriesExec { label_source: RawLabelSource, /// The ordering the input frames guarantee. ordering: SeriesOrdering, - /// The position of the tree's input node, where every raw chunk a partition reads enters. - input_path: NodePath, + /// Which of the tree's inputs every raw chunk a partition reads enters at. + input_id: InputId, /// Where each output column comes from. classification: Arc<[OutputColumn]>, /// The output column holding the grid timestamps, kept so this node can be rebuilt over a new @@ -767,8 +772,8 @@ pub struct FusedSeriesExec { /// /// Plan-level rather than per-partition: see [`LabelSets`]. labels: SharedLabelSets, - /// Per-node metrics, registered by each execution of the tree. - metric: ExecutionPlanMetricsSet, + /// Runtime roots from executions of the tree, retained for fresh metrics exports. + runtimes: Mutex>, /// This node's partitioning, emission and boundedness. properties: Arc, } @@ -799,12 +804,7 @@ impl FusedSeriesExec { // Building a state is what proves the runtime executes this tree: a node kind it does not // run yet is refused here, so a plan that is built is a plan that starts. The throwaway // state is dropped at once, returning its (unbounded) reservations. - let _ = NativeExecutionState::try_new( - &tree, - Arc::new(UnboundedMemoryPool::default()), - &ExecutionPlanMetricsSet::new(), - 0, - )?; + let _ = NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()))?; let classification = classify_output(&output_schema, &tree_input.schema, &tags, &time_column)?; // Read here rather than below, because whether the split below can happen decides whether @@ -867,14 +867,14 @@ impl FusedSeriesExec { series_schema: tree_input.schema, label_source, ordering, - input_path: tree_input.path, + input_id: tree_input.input_id, classification: classification.into(), time_column, tags: tags.into(), input, output_schema, labels, - metric: ExecutionPlanMetricsSet::new(), + runtimes: Mutex::new(Vec::new()), properties, }) } @@ -987,15 +987,18 @@ impl ExecutionPlan for FusedSeriesExec { let state = NativeExecutionState::try_new( &self.tree, Arc::clone(&context.runtime_env().memory_pool), - &self.metric, - partition, )?; + let runtime = state.runtime_tree(); + self.runtimes + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push((partition, runtime)); // What the label sets hold is charged to the reservation attached above, so nothing is // reserved per stream: the entries outlive every stream and are released with the plan. Ok(Box::pin(FusedSeriesStream { input: Some(input), executor: Some(NativeExecutor::new(state)), - input_path: self.input_path.clone(), + input_id: self.input_id, classification: Arc::clone(&self.classification), tags: Arc::clone(&self.tags), output_schema: self.output_schema.clone(), @@ -1006,7 +1009,15 @@ impl ExecutionPlan for FusedSeriesExec { } fn metrics(&self) -> Option { - Some(self.metric.clone_inner()) + let runtimes = self.runtimes.lock().unwrap_or_else(|e| e.into_inner()); + let mut metrics = MetricsSet::new(); + for (partition, runtime) in runtimes.iter() { + let runtime = runtime.lock().unwrap_or_else(|e| e.into_inner()); + for metric in runtime.export_metrics(*partition).iter() { + metrics.push(Arc::clone(metric)); + } + } + Some(metrics) } } @@ -1368,7 +1379,7 @@ pub struct FusedSeriesStream { /// The execution of this node's tree, or `None` once it has been closed. executor: Option, /// The position of the tree's input node, which is where every chunk read enters. - input_path: NodePath, + input_id: InputId, /// Where each output column comes from. classification: Arc<[OutputColumn]>, /// The tag column names, indexed by `OutputColumn::Tag`. @@ -1395,7 +1406,7 @@ impl FusedSeriesStream { }; // The chunk enters at the tree's input node by position: a position is how the runtime // addresses a node, and a tree that gains a second input names them the same way. - executor.push_raw_at(&self.input_path, chunk)?; + executor.push_raw(self.input_id, chunk)?; let mut settled = Vec::new(); while let Some(chunk) = executor.next_chunk()? { settled.push(chunk); diff --git a/src/promql/src/extension_plan/fused_series/tests.rs b/src/promql/src/extension_plan/fused_series/tests.rs index c3837187474..c79305c77dd 100644 --- a/src/promql/src/extension_plan/fused_series/tests.rs +++ b/src/promql/src/extension_plan/fused_series/tests.rs @@ -54,7 +54,7 @@ use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_optimizer::ensure_requirements::EnsureRequirements; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue, MetricsSet}; +use datafusion::physical_plan::metrics::{MetricValue, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties, RecordBatchStream, SendableRecordBatchStream, collect, displayable, @@ -995,13 +995,8 @@ async fn a_runtime_error_after_rows_releases_the_input() { fn two_differently_named_series_collide_once_rate_drops_the_name() { let tree = rate_tree(); let mut execution = NativeExecutor::new( - NativeExecutionState::try_new( - &tree, - Arc::new(UnboundedMemoryPool::default()), - &ExecutionPlanMetricsSet::new(), - 0, - ) - .expect("the runtime executes a rate tree"), + NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default())) + .expect("the runtime executes a rate tree"), ); let samples = series_samples(1.0); execution @@ -1218,6 +1213,48 @@ async fn a_series_spanning_chunks_keeps_absolute_positions_and_closes_once() { .expect("the node reports its runtime's metrics"); assert_eq!(1, count(&metrics, "native.root.input.completed_series")); assert_eq!(1, count(&metrics, "native.root.completed_series")); + + // Exporting a snapshot repeatedly does not register another copy of the runtime metrics. + let repeated = exec.metrics().expect("metrics can be exported repeatedly"); + assert_eq!(metrics.iter().count(), repeated.iter().count()); + assert_eq!(1, count(&repeated, "native.root.input.completed_series")); + assert_eq!(1, count(&repeated, "native.root.completed_series")); +} + +/// Dropping a partially consumed execution releases its live reservation but preserves its peak. +#[tokio::test] +async fn cancelling_after_partial_input_clears_current_bytes_and_keeps_peak() { + let schema = frame_schema(); + let partial = frame(&series_chunk("web-1", &series_samples(1.0)[..4], false)); + let exec = plan( + rate_tree(), + staged(schema.clone(), move || { + Box::pin(StagedStream::new(schema.clone(), vec![partial.clone()]).pending_after(1)) + }), + ) + .expect("the runtime executes a rate tree"); + let pool = Arc::new(GreedyMemoryPool::new(1 << 20)); + let mut stream = exec + .execute(0, task_context(&pool)) + .expect("the node executes"); + + // One unclosed series chunk is accepted; the source then parks before another arrives. + assert!(futures::poll!(stream.next()).is_pending()); + drop(stream); + + let metrics = exec.metrics().expect("the runtime metrics are exported"); + let current = metrics + .sum_by_name("native.root.current_reserved_bytes") + .expect("the current-bytes gauge is exported"); + let peak = metrics + .sum_by_name("native.p0.root.peak_reserved_bytes") + .expect("the peak-bytes gauge is exported"); + assert!(matches!(current, MetricValue::Gauge { gauge, .. } if gauge.value() == 0)); + assert!(matches!(peak, MetricValue::Gauge { gauge, .. } if gauge.value() > 0)); + assert!( + pool.reserved() > 0, + "plan-level label-set charge is independent" + ); } /// A tree the runtime does not execute is refused when the execution plan is built. diff --git a/src/promql/src/native/aggregate.rs b/src/promql/src/native/aggregate.rs index 975dfb6c3c2..e3fad1a572c 100644 --- a/src/promql/src/native/aggregate.rs +++ b/src/promql/src/native/aggregate.rs @@ -38,8 +38,7 @@ use crate::aggregate::{AggregateOp, FloatAggregator, Grouping}; use crate::evaluated::EvaluatedChunk; use crate::grid_series::series_bytes; use crate::map::validate_float_series; -use crate::native::execute::{NodeBuffer, evaluated_bytes}; -use crate::native::metrics::NodePath; +use crate::native::execute::{evaluated_bytes, NodeBuffer}; use crate::native::plan::{ AggregateKind, AggregateStage, BoundAggregate, Grouping as BoundGrouping, }; @@ -152,7 +151,6 @@ impl AggregateRuntime { /// what makes a group's answer its whole group's answer once the input ends. pub(crate) fn accumulate( &mut self, - path: &NodePath, chunk: EvaluatedChunk, buffer: &mut NodeBuffer, pending: usize, @@ -161,8 +159,12 @@ impl AggregateRuntime { // another schema; the states are indexed by absolute position on this node's grid, so a // chunk of another field count or another grid would be folded into the wrong places. validate_float_series(&self.schema, &chunk.series) - .map_err(|error| folded_chunk(path, &self.describe(), error))?; - let accumulator = self.accumulator.as_mut().ok_or_else(|| no_fold(path))?; + .map_err(|error| folded_chunk(&self.describe(), error))?; + let aggregate = self.describe(); + let accumulator = self + .accumulator + .as_mut() + .ok_or_else(|| no_fold(&aggregate))?; // Keep the chunk covered while the fold decides whether it needs a new group or block. accumulator.update_budgeted(&chunk.series, |fold_bytes| { buffer.resize(pending.saturating_add(fold_bytes)) @@ -183,7 +185,6 @@ impl AggregateRuntime { /// a fold whose states are gone. pub(crate) fn compute( &mut self, - path: &NodePath, buffer: &mut NodeBuffer, queue_bytes: usize, ) -> Result>> { @@ -191,13 +192,13 @@ impl AggregateRuntime { return Ok(None); } if self.accumulator.is_none() { - return Err(no_fold(path)); + return Err(no_fold(&self.describe())); } self.computed = true; let output = self .accumulator .take() - .ok_or_else(|| no_fold(path))? + .ok_or_else(|| no_fold(&self.describe()))? .finish_budgeted(|total| buffer.resize(total.saturating_add(queue_bytes)))?; let output_bytes = output.iter().fold(0usize, |bytes, series| { bytes.saturating_add(series_bytes(series)) @@ -225,19 +226,19 @@ impl AggregateRuntime { /// kernel saying it does not compute this shape at all -- a native histogram in a float aggregate, /// for instance -- and a caller that falls back to another planner on `NotImplemented` still can. /// Wrapping that one in an `Execution` would deny it. -fn folded_chunk(path: &NodePath, aggregate: &str, error: DataFusionError) -> DataFusionError { +fn folded_chunk(aggregate: &str, error: DataFusionError) -> DataFusionError { match error { DataFusionError::NotImplemented(_) => error, error => DataFusionError::Execution(format!( - "the `{aggregate}` aggregate at {path} folds chunks bound to its own schema, and this \ + "the `{aggregate}` aggregate folds chunks bound to its own schema, and this \ one is not: {error}" )), } } /// The error for a fold that has already been finished. -fn no_fold(path: &NodePath) -> DataFusionError { +fn no_fold(aggregate: &str) -> DataFusionError { DataFusionError::Execution(format!( - "the aggregate at {path} already finished its fold, so it takes no more input" + "the `{aggregate}` aggregate already finished its fold, so it takes no more input" )) } diff --git a/src/promql/src/native/display.rs b/src/promql/src/native/display.rs index 7d578f884ea..4b1cb89d4aa 100644 --- a/src/promql/src/native/display.rs +++ b/src/promql/src/native/display.rs @@ -14,10 +14,9 @@ //! One line per node of a [`NativeTree`]. //! -//! The position a node is shown at is derived by [`metrics::walk`], the same traversal that names -//! the node's metrics, so a node cannot be shown at one place and measured at another. The -//! display is an observation of a compiled tree: it runs nothing, and every line reads as the node -//! it is, making no claim about whether that node could run. +//! The tree is displayed by recursively visiting its bound expressions. The display is an +//! observation of a compiled tree: it runs nothing, and every line reads as the node it is, +//! making no claim about whether that node could run. use std::fmt::{self, Write}; use std::sync::Arc; @@ -27,7 +26,6 @@ use datatypes::series::{MetricName, SeriesOrdering}; use crate::histogram_eval::HistogramFunction; use crate::map::MathOp; -use crate::native::metrics::{self, NodePath}; use crate::native::plan::{ AggregateKind, AggregateStage, BoundVectorOp, BoundWindow, Grouping, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, @@ -39,17 +37,6 @@ use crate::vector_match::{BinaryOp, Cardinality, FillValues, MatchingLabels}; /// The header line of every tree. const HEADER: &str = "NativeTree"; -/// How deep `path` is: the root is at depth zero, its input at one. -fn depth(path: &NodePath) -> usize { - path.as_str().matches('.').count() -} - -/// The last segment of `path`, which is the name of the node in its parent. -fn name(path: &NodePath) -> &str { - // `NodePath` only ever appends `.segment`, so the last segment is never empty. - path.as_str().rsplit('.').next().unwrap_or(path.as_str()) -} - /// A grid as `/x`. fn grid_name(grid: &EvalGrid) -> String { let spec = grid.spec(); @@ -346,29 +333,39 @@ fn kind(expr: &NativeExpr) -> &'static str { } } -/// The line a node is shown as, without its indentation. -fn line(node: &NativeNode, path: &NodePath) -> String { - format!("{} {} {}", name(path), kind(&node.expr), describe(node)) +/// One recursively displayed node and its children. +fn display_node( + f: &mut fmt::Formatter<'_>, + node: &NativeNode, + depth: usize, + edge: &str, +) -> fmt::Result { + f.write_char('\n')?; + // The root sits one level in from the header, and each child one level below its consumer. + for _ in 0..depth + 1 { + f.write_str(" ")?; + } + write!(f, "{edge} {} {}", kind(&node.expr), describe(node))?; + match &node.expr { + NativeExpr::Input(_) | NativeExpr::Scalar(_) => {} + NativeExpr::Instant { input, .. } + | NativeExpr::Window { input, .. } + | NativeExpr::Map { input, .. } + | NativeExpr::Aggregate { input, .. } + | NativeExpr::Vector { input, .. } => display_node(f, input, depth + 1, "input")?, + NativeExpr::Binary { left, right, .. } => { + display_node(f, left, depth + 1, "left")?; + display_node(f, right, depth + 1, "right")?; + } + } + Ok(()) } impl fmt::Display for NativeTree { /// One line per node, outermost first, indented two spaces per depth below the header. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(HEADER)?; - let mut lines = Vec::new(); - metrics::walk(self.root(), &mut |node, path| { - lines.push((depth(path), line(node, path))); - }); - for (depth, line) in lines { - f.write_char('\n')?; - // The root sits one level in from the header, and each input one level below its - // consumer, so the shape of the tree is the shape of the output. - for _ in 0..depth + 1 { - f.write_str(" ")?; - } - f.write_str(&line)?; - } - Ok(()) + display_node(f, self.root(), 0, "root") } } diff --git a/src/promql/src/native/execute.rs b/src/promql/src/native/execute.rs index eb7b3dcca57..1a9325173e1 100644 --- a/src/promql/src/native/execute.rs +++ b/src/promql/src/native/execute.rs @@ -14,83 +14,17 @@ //! Running a bound tree, node by node. //! -//! [`NativeExecutionState`] owns the run-time state one execution of a [`NativeTree`] needs, keyed -//! by the position [`crate::native::metrics::walk`] gives each node. The tree is what the compiler -//! produced and carries no state of its own, so a second execution builds a second state from the -//! same tree and the two never share a buffer, an evaluator or a memory reservation. -//! -//! [`NativeExecutor`] drives that state one chunk at a time. Nodes hand each other -//! [`EvaluatedChunk`]s: a source hands over raw samples, and everything above a window hands over -//! the evaluator's own chunk type. No row batch and no frame is materialised on an internal edge, -//! so a chunk a window produced is fed to the next node as it is. -//! -//! Every window owns a [`RangeEvaluator`], which is what keeps its input grid and its output grid -//! its own: `Window -> Map -> Window` evaluates the inner window on the inner grid and the outer -//! window on the outer grid, and the map in between carries the inner grid through untouched. The -//! outer window reads the inner result as the samples it is, not as something to be resampled onto -//! the outer grid first. -//! -//! An instant selection owns no evaluator, because [`select_instants`] takes a complete source -//! axis: the node buffers the chunks of the series it is selecting until the chunk that carries -//! `is_last`, selects the last sample at or before each grid point of it, and hands the series over -//! as one chunk addressed on its own grid. Nothing is decided for a series before it ends, which is -//! the price of the kernel taking one axis rather than a chunk of one. -//! -//! A vector–vector binary node is the only kind that reads two inputs: it retains the chunks of both -//! operands and runs the matching kernel once the input has ended, because the operation pairs two -//! whole vectors point by point and nothing is decided for a point until both sides are complete. -//! The chunks of one logical series are assembled into the single series the kernel matches before it -//! runs, so a series that arrived in many chunks is one candidate, not one per chunk. A vector operation -//! (`topk`, `limitk`, `limit_ratio`) reads its one input the same way, because which samples it -//! keeps at a point depends on every series present there. -//! -//! A constant scalar is a value node rather than a stream: it has no input and hands over no chunk, -//! because what it produces is the one value it was bound to and not a series. A binary node with a -//! scalar operand therefore reads one input — the vector side — and applies the operation to each -//! chunk it is handed as it arrives: the scalar is read off the node's binding rather than pulled, -//! and no chunk of the vector is retained, because one vector value pairs with one constant and -//! nothing is decided by what comes after it. That broadcast is a different mode from the vector -//! matching above and keeps neither operand: what it hands out is the chunk it transformed. -//! -//! A node kind this runtime does not execute yet is rejected when the execution is built, naming -//! the kind and its position. That is an aggregate, and a binary node whose two operands are -//! neither two evaluated vectors nor one evaluated vector and one constant. An unsupported tree -//! therefore never starts, rather than answering half a query or panicking halfway through one. -//! -//! A node that rewrites labels — a window drops the metric name and the empty-valued labels, a map -//! drops the name or rewrites a label, and a binary node drops it for arithmetic and for a `bool` comparison — can turn -//! two distinct input series into one output label set. That pair is legal when the two hold values -//! at disjoint positions, which is exactly what -//! [`crate::grid_series::merge_same_labelset`] merges: `is_last` closes the source series whose -//! data the chunk carries, not the label set it shares with another series, so a second series may -//! fill the positions the first one left empty. The node that rewrote the labels keeps which -//! positions each output label set already occupies; a second series that puts a value where the -//! same field already holds one is refused there — before a node above reads it — while a chunk -//! that continues a set where it ended, or fills positions the set has not occupied, is handed on. -//! -//! A label set reachable by more than one series needs one more guarantee: a window reads one -//! series to its `is_last` before the next one, so the per-series state of its evaluator ends -//! before a second series onto the same set arrives. The window immediately above such a rewriting -//! node therefore collects its whole input, merges each canonical label set once with -//! [`crate::grid_series::merge_same_labelset`], and only then evaluates it: every logical series -//! reaches its evaluator once and whole, and the pieces that would have been separate streams -//! become one. Such a window reports at the end of its input instead of as its input arrives. A -//! window directly over a source, and a map at the root, are not affected and keep streaming their -//! chunks on as they produce them. -//! -//! What leaves the tree is therefore a piece stream, not a series collection: a chunk's `is_last` -//! says the source series it carries ended. [`NativeExecutor::finish`] performs the merge for every -//! affected window inside the tree. A caller that needs one series per canonical label set from the -//! root itself — a downstream window outside this runtime, or a protocol that reports one series — -//! collects the root's chunks and merges them with the same helper. +//! Each execution owns a recursive runtime tree; the compiled [`NativeTree`] remains immutable and +//! can be used to build independent executions. Per-node metric handles are exported only when +//! requested, using temporary names derived by traversing the runtime tree. use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use datafusion::error::{DataFusionError, Result}; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; -use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet}; use datatypes::arrow::array::Float64Array; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; use datatypes::series::histogram::HistogramView; @@ -109,12 +43,12 @@ use crate::grid_series::{ use crate::histogram_eval::{evaluate_histograms, map_native_histogram}; use crate::instant_eval::select_instants; use crate::map::{ - ArithmeticOp, Comparison, ComparisonOp, ScalarOperand, drop_metric_name, map_arithmetic, - map_compare, map_math, + ArithmeticOp, Comparison, ComparisonOp, MathOp, ScalarOperand, drop_metric_name, + map_arithmetic, map_compare, map_math, }; use crate::native::aggregate::AggregateRuntime; use crate::native::batch::EvaluatedChunk; -use crate::native::metrics::{NodeMetrics, NodePath}; +use crate::native::metrics::NodeMetrics; use crate::native::plan::{ BoundInstant, BoundScalar, BoundVectorOp, Grouping, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, @@ -334,56 +268,25 @@ fn buffer_bytes(buffer: &Buffer) -> usize { buffer.capacity().max(buffer.len()) } -/// Which node kinds the runtime executes. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum NodeTag { - /// Forwards the chunks it is given. - Input, - /// A constant scalar: the value its consumer broadcasts, and no chunk of its own. - /// - /// It is a value node and not a stream. Its output type is [`NativeType::Scalar`], and the - /// runtime produces evaluated chunks only, so the node hands nothing over: what its consumer - /// reads is the constant bound to it, which is why it has no input and no run-time state. A - /// `pull` of it answers `None` for exactly that reason, and that is not a stall to fix. - Scalar, - /// Buffers one series and selects the last sample at or before each of its grid points. - Instant, - /// Feeds its input to its own [`RangeEvaluator`]. - Window, - /// Applies point-wise operations to its input chunks. - Map, - /// Matches two evaluated vectors and applies one operation to every matched pair of series, or - /// broadcasts one constant over the chunks of one evaluated vector. - Binary, - /// Folds every series of the input into one series per group, when the input ends. - Aggregate, - /// Keeps some samples of its whole input vector at each grid point. - Vector, -} +/// Which of an execution's inputs a caller's chunk belongs to. +/// +/// An input is a leaf of the tree, and a tree of unary nodes has one while a binary node adds one +/// per operand, so a caller has to say which leaf it is feeding. The id is the leaf's position +/// among the leaves counted left to right, which is stable for one tree and means nothing across +/// trees: it is a routing handle, not an identity a node carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct InputId(usize); -impl NodeTag { - /// What this tag is called in a rejection. - fn describe(self) -> &'static str { - match self { - Self::Input => "input", - Self::Scalar => "scalar", - Self::Instant => "instant selection", - Self::Window => "window", - Self::Map => "map", - Self::Binary => "binary", - Self::Aggregate => "aggregate", - Self::Vector => "vector operation", - } +impl InputId { + /// The input at `index`, counted left to right. + pub fn new(index: usize) -> Self { + Self(index) } -} -/// One node of the flattened tree. -#[derive(Clone, Copy, Debug)] -struct NodeSpec { - tag: NodeTag, - /// The index of this node's input, which is the node the build pass reaches next: the one - /// input of a unary node, the left operand of a binary node, and none for an input node. - input: Option, + /// The position this id names. + pub fn index(self) -> usize { + self.0 + } } /// The series whose first chunk this buffer received and whose closing chunk it has not seen. @@ -399,7 +302,7 @@ struct OpenSeries { last_position: Option, } -/// The input a window with a [`NodeRuntime::merge`] collects until its input ends. +/// The input a window with a [`RuntimeOp::Window`] collects until its input ends. /// /// Such a window cannot evaluate the series under one canonical label set one after another: its /// evaluator closes a series on the chunk carrying `is_last`, and the points a window across the @@ -454,7 +357,7 @@ impl MergeInput { /// /// A chunk carrying `is_last` closes the open series, whether it holds rows or none at all, and /// nothing is remembered of it: the next chunk starts a series whose identity is its own. - fn check(&mut self, path: &NodePath, series: &Series, is_last: bool) -> Result<()> { + fn check(&mut self, series: &Series, is_last: bool) -> Result<()> { let on_grid = match (&self.schema.eval_grid, series.time.grid()) { (Some(input), Some(grid)) => input.spec() == grid.spec(), _ => false, @@ -466,33 +369,35 @@ impl MergeInput { .iter() .any(|field| field.len() != series.time.len()) { - return Err(DataFusionError::Execution(format!( - "the window at {path} was handed an evaluated series that does not match its input \ - schema" - ))); + return Err(DataFusionError::Execution( + "the window was handed an evaluated series that does not match its input schema" + .to_string(), + )); } let (first, last) = (series.time.first_position(), series.time.last_position()); if !is_last && first.is_none() { - return Err(DataFusionError::Execution(format!( - "the window at {path} was handed an evaluated series that holds no position before \ - the last run of its series" - ))); + return Err(DataFusionError::Execution( + "the window was handed an evaluated series that holds no position before the last \ + run of its series" + .to_string(), + )); } if let Some(open) = &self.open && open.metric != series.metric { - return Err(DataFusionError::Execution(format!( - "the window at {path} was handed an evaluated series that changed identity before \ - its last run" - ))); + return Err(DataFusionError::Execution( + "the window was handed an evaluated series that changed identity before its last \ + run" + .to_string(), + )); } if let Some(previous) = self.open.as_ref().and_then(|open| open.last_position) && let Some(first) = first && first <= previous { return Err(DataFusionError::Execution(format!( - "the window at {path} was handed evaluated series runs that are not increasing: \ - {first} after {previous}" + "the window was handed evaluated series runs that are not increasing: {first} after \ + {previous}" ))); } self.open = if is_last { @@ -534,9 +439,13 @@ pub(super) struct NodeBuffer { } impl NodeBuffer { - /// Registers a reservation for the node at `path` with `pool`. - fn new(path: &NodePath, metrics: Arc, pool: &Arc) -> Self { - let consumer = MemoryConsumer::new(format!("native.{}", path.as_str())); + /// Registers a reservation for the node called `name` with `pool`. + /// + /// The name is what the operator is — `native.input`, `native.window` — rather than where it + /// sits in the tree: a consumer is named once, when the node owning it is built, so a tree + /// executed twice registers the same consumers again instead of a second set of them. + fn new(name: &'static str, metrics: Arc, pool: &Arc) -> Self { + let consumer = MemoryConsumer::new(name); Self { metrics, reservation: consumer.register(pool), @@ -582,8 +491,8 @@ impl NodeBuffer { /// The positions one node has handed out, per output label set and field. /// -/// A node that rewrites labels — a `Map` dropping the metric name or rewriting a label, a `Window` -/// dropping the name and the empty-valued labels — can turn two input series into one label set, and the node above +/// A node that rewrites labels — a `Map` dropping the metric name, a `Window` dropping the name +/// and the empty-valued labels — can turn two input series into one label set, and the node above /// reads the two chunks as one series. What is kept here is which positions of each field were /// already handed out under each label set, so a second series onto a set can be refused before /// the node above evaluates it. @@ -665,14 +574,14 @@ impl Emitted { /// field at one position, so that is the only thing refused here. `is_last` is not consulted: it /// closes the source series the chunk carries, not the label set, and a later series may still /// fill the positions that one left empty. - fn occupy(&mut self, path: &NodePath, chunk: &EvaluatedChunk) -> Result<()> { + fn occupy(&mut self, chunk: &EvaluatedChunk) -> Result<()> { let labels = label_set(&chunk.series.metric); let occupied: Vec> = chunk .series .fields .iter() .enumerate() - .map(|(index, field)| occupied_runs(path, index, field, &chunk.series.time)) + .map(|(index, field)| occupied_runs(index, field, &chunk.series.time)) .collect::>()?; let empty = occupied.iter().all(Vec::is_empty); if let Some(history) = self.sets.get(&labels) { @@ -686,7 +595,7 @@ impl Emitted { let at = stored.partition_point(|stored| stored.end <= run.start); let previous = stored.get(at).filter(|stored| stored.overlaps(*run)); if let Some(previous) = previous { - return Err(occupied_position(path, &labels, index, run, previous)); + return Err(occupied_position(&labels, index, run, previous)); } } } @@ -744,12 +653,7 @@ impl Emitted { /// The axis is what says how many rows there are, so every field it reads has to hold one sample per /// row. [`Series`] keeps its fields public, so a chunk built by struct literal can carry fewer, and /// that is refused here rather than read past the end of the field. -fn occupied_runs( - path: &NodePath, - index: usize, - field: &Samples, - time: &TimeAxis, -) -> Result> { +fn occupied_runs(index: usize, field: &Samples, time: &TimeAxis) -> Result> { let mut runs: Vec = Vec::new(); let mut open: Option = None; let mut previous: Option = None; @@ -768,7 +672,7 @@ fn occupied_runs( } let sample = cursor .at(row) - .ok_or_else(|| misaligned_field(path, index, time, field))?; + .ok_or_else(|| misaligned_field(index, time, field))?; let present = !matches!(sample, SampleRef::Missing); match (present, open) { (true, None) => open = Some(position), @@ -799,16 +703,10 @@ fn occupied_runs( /// [`Series`] keeps its fields public, so a chunk built by struct literal — by a caller of this crate /// as much as by a mistake here — can carry fewer samples than the axis places rows, and then the row /// the axis names has nothing to read. -fn misaligned_field( - path: &NodePath, - index: usize, - time: &TimeAxis, - field: &Samples, -) -> DataFusionError { +fn misaligned_field(index: usize, time: &TimeAxis, field: &Samples) -> DataFusionError { DataFusionError::Execution(format!( - "the node at {path} was handed a series whose axis and fields are misaligned: the axis \ - places {} rows, but field {index} holds {} samples, so the axis names a row the field does \ - not have", + "a series was handed to the node whose axis and fields are misaligned: the axis places {} \ + rows, but field {index} holds {} samples, so the axis names a row the field does not have", time.len(), field.len(), )) @@ -1026,14 +924,14 @@ impl InstantRuntime { /// /// `None` means the series is still open: the selection reads one whole axis, so nothing is /// decided for a series until the chunk carrying `is_last` arrives. - fn push(&mut self, path: &NodePath, chunk: RawSeriesChunk) -> Result> { + fn push(&mut self, chunk: RawSeriesChunk) -> Result> { if let Some(open) = self.chunks.first() && (open.primary_key.as_slice() != chunk.primary_key.as_slice() || open.metric != chunk.metric) { // One axis is gathered before the kernel reads any of it, so a chunk of another series // has nowhere to go: it would be selected as though its samples were the open series'. - return Err(series_changed(path)); + return Err(series_changed()); } let closing = chunk.is_last; self.held = self.held.saturating_add(raw_bytes(&chunk)); @@ -1041,12 +939,12 @@ impl InstantRuntime { if !closing { return Ok(None); } - self.gather(path).map(Some) + self.gather().map(Some) } /// Selects the instants of the buffered series, hands it over closed, and drops the buffer. - fn gather(&mut self, path: &NodePath) -> Result { - let chunk = self.select(path)?; + fn gather(&mut self) -> Result { + let chunk = self.select()?; // The selection copied what it needed out of the buffered chunks, so a series that has been // answered must not keep charging the pool for them. self.chunks.clear(); @@ -1056,13 +954,13 @@ impl InstantRuntime { } /// The last sample at or before each grid point of the buffered series, as one chunk. - fn select(&mut self, path: &NodePath) -> Result { + fn select(&mut self) -> Result { let Some(first) = self.chunks.first() else { - return Err(DataFusionError::Execution(format!( - "the instant selection at {path} holds no series to select" - ))); + return Err(DataFusionError::Execution( + "the instant selection holds no series to select".to_string(), + )); }; - let metric = first.metric.clone().ok_or_else(|| no_labels(path))?; + let metric = first.metric.clone().ok_or_else(no_labels)?; // The buffered chunks are one series, so they are read as the one axis the selection takes. let time = buffered_time(&self.chunks)?; let fields = buffered_fields(&self.chunks, self.schema.fields.len())?; @@ -1093,7 +991,7 @@ impl InstantRuntime { // Two grid points can select one source row, and the row is read once for both. if read != Some(index) { for (slot, cursor) in row.iter_mut().zip(&mut cursors) { - let sample = cursor.at(index).ok_or_else(|| missing_row(path, index))?; + let sample = cursor.at(index).ok_or_else(|| missing_row(index))?; *slot = owned(sample)?; } read = Some(index); @@ -1193,7 +1091,7 @@ impl RetainedInput { /// An input hands over one series at a time, each chunk of it after the rows the series already /// reached; a chunk that starts another series while one is open, or goes back inside the open /// one, is refused before it is retained. - fn retain(&mut self, chunk: EvaluatedChunk, reader: Reader<'_>) -> Result<()> { + fn retain(&mut self, chunk: EvaluatedChunk, reader: Reader) -> Result<()> { // An empty chunk closes a series without holding a row, so it says nothing about where that // series ended: only a chunk that carries rows can be read as going backwards. let (first, last) = ( @@ -1232,7 +1130,7 @@ impl RetainedInput { /// An input hands over one series at a time, so the only series that can be open is the last /// one it handed over: a chunk that did not say `is_last` leaves it open, and nothing came after /// it to close it. - fn ensure_closed(&self, reader: Reader<'_>) -> Result<()> { + fn ensure_closed(&self, reader: Reader) -> Result<()> { match &self.open { Some((metric, _)) => Err(DataFusionError::Execution(format!( "the input of {} ended before the last chunk of a series: {} handed over \ @@ -1351,7 +1249,7 @@ impl RetainedInput { /// the kernel matches and reads, so it must be on the schema's grid and hold as many rows as its /// axis places, which is what [`Series::try_new`] checks. What it allocates is /// [`Self::assembly_bytes`], which a caller reserves before this runs. - fn assemble(&self, schema: &SeriesSchema, reader: Reader<'_>) -> Result> { + fn assemble(&self, schema: &SeriesSchema, reader: Reader) -> Result> { let grid = schema.eval_grid.clone().ok_or_else(|| { let (owner, input) = (reader.owner(), reader.input()); DataFusionError::Execution(format!( @@ -1448,25 +1346,25 @@ fn side_name(index: usize) -> &'static str { /// The node a [`RetainedInput`] belongs to, which its messages name; formatted only when one is. #[derive(Clone, Copy)] -enum Reader<'a> { - /// The operand at `side` of the binary node at `path`. - Operand { path: &'a NodePath, side: usize }, - /// The input of the vector operation at the path. - Vector(&'a NodePath), +enum Reader { + /// The operand at `side` of a binary node. + Operand { side: usize }, + /// The input of a vector operation. + Vector, } -impl Reader<'_> { +impl Reader { fn owner(self) -> String { match self { - Self::Operand { path, .. } => format!("the binary node at {path}"), - Self::Vector(path) => format!("the vector operation at {path}"), + Self::Operand { .. } => "the binary node".to_string(), + Self::Vector => "the vector operation".to_string(), } } fn input(self) -> String { match self { - Self::Operand { side, .. } => format!("its {} operand", side_name(side)), - Self::Vector(_) => "its input".to_string(), + Self::Operand { side } => format!("its {} operand", side_name(side)), + Self::Vector => "its input".to_string(), } } } @@ -1499,12 +1397,12 @@ impl BinaryRuntime { } /// The logical series both operands closed, ready for the kernel to match. - fn assemble(&self, path: &NodePath) -> Result<[Vec; 2]> { + fn assemble(&self) -> Result<[Vec; 2]> { let [left, right] = &self.operands; let [left_schema, right_schema] = &self.schemas; Ok([ - left.assemble(left_schema, Reader::Operand { path, side: 0 })?, - right.assemble(right_schema, Reader::Operand { path, side: 1 })?, + left.assemble(left_schema, Reader::Operand { side: 0 })?, + right.assemble(right_schema, Reader::Operand { side: 1 })?, ]) } @@ -1513,46 +1411,45 @@ impl BinaryRuntime { /// It is read from the operand before the chunk is retained, so the figure is the one the /// operand grows by: what the chunk holds on its own and the room its end takes in the operand's /// boundary list. - fn pending(&self, path: &NodePath, side: usize, chunk: &EvaluatedChunk) -> Result { + fn pending(&self, side: usize, chunk: &EvaluatedChunk) -> Result { self.operands .get(side) .map(|operand| operand.pending(chunk)) - .ok_or_else(|| no_binary(path)) + .ok_or_else(no_binary) } /// Reports an operand that ended inside a series. /// /// The kernel is not run over such an operand: it would answer the pairs of a series whose /// remaining rows were never read. - fn closed(&self, path: &NodePath) -> Result<()> { + fn closed(&self) -> Result<()> { for (side, operand) in self.operands.iter().enumerate() { - operand.ensure_closed(Reader::Operand { path, side })?; + operand.ensure_closed(Reader::Operand { side })?; } Ok(()) } /// Retains one chunk of the operand at `side`, checking it against that operand's binding. - fn retain(&mut self, path: &NodePath, side: usize, chunk: EvaluatedChunk) -> Result<()> { + fn retain(&mut self, side: usize, chunk: EvaluatedChunk) -> Result<()> { // What a bound tree says an operand produces is not proof of what a chunk carries: a // `Series` keeps its fields public, so a chunk built by struct literal can carry another // number of fields, or fewer samples than its axis places rows. The kernel's own check is - // run here, before the chunk enters the buffer, and the position and side are added to what - // it reports. + // run here, before the chunk enters the buffer, and the side is added to what it reports. let Some(schema) = self.schemas.get(side) else { - return Err(no_binary(path)); + return Err(no_binary()); }; validate_binary_operand(schema.as_ref(), std::slice::from_ref(&chunk.series)).map_err( |error| { DataFusionError::Execution(format!( - "the binary node at {path} rejected its {} operand: {error}", + "the binary node rejected its {} operand: {error}", side_name(side) )) }, )?; let Some(operand) = self.operands.get_mut(side) else { - return Err(no_binary(path)); + return Err(no_binary()); }; - operand.retain(chunk, Reader::Operand { path, side }) + operand.retain(chunk, Reader::Operand { side }) } /// Runs the kernel over both operands and returns the series it produced. @@ -1568,25 +1465,23 @@ impl BinaryRuntime { /// phase of its walk, and in the blocks a growing output buffer is reserved in -- rather than per /// grid point, and the total is resized as a phase's storage is released. It keeps covering the /// returned results until the caller queues them. - fn compute(&mut self, path: &NodePath, buffer: &mut NodeBuffer) -> Result>> { + fn compute(&mut self, buffer: &mut NodeBuffer) -> Result>> { // The kernel reads the operands once and drops them, so a second call has nothing to read // and nothing left to produce: what it produced the first time is in the node's queue, and // the reservation is already the queue's. if self.computed { return Ok(None); } - self.closed(path)?; + self.closed()?; let [left_schema, right_schema] = &self.schemas; let retained = self.retained_bytes(); // The assembled series are live while the kernel reads them and share the buffers the // retained chunks hold, so what they add is metadata, and it is reserved before it exists: // the assembly allocates as it goes, and a figure read from it afterwards would be a figure // the pool never admitted. - let assembly = self - .assembly_bytes() - .ok_or_else(|| reservation_overflow(path))?; + let assembly = self.assembly_bytes().ok_or_else(reservation_overflow)?; buffer.grow(assembly)?; - let assembled = match self.assemble(path) { + let assembled = match self.assemble() { Ok(assembled) => assembled, Err(error) => { // Nothing the assembly allocated outlives this call, so the node falls back to what @@ -1615,7 +1510,7 @@ impl BinaryRuntime { let bytes = retained .checked_add(assembly) .and_then(|held| held.checked_add(workspace)) - .ok_or_else(|| reservation_overflow(path))?; + .ok_or_else(reservation_overflow)?; buffer.resize(bytes) }, ) @@ -1671,18 +1566,18 @@ impl VectorRuntime { /// The reservation covers the retained chunks, the assembled series and a rough estimate of the kernel's workspace and of the series it rebuilds /// ([`vector_workspace_bytes`]). The kernels do not report their workspace as they allocate, /// so the estimate is reserved once, before the kernel runs. - fn compute(&mut self, path: &NodePath, buffer: &mut NodeBuffer) -> Result>> { + fn compute(&mut self, buffer: &mut NodeBuffer) -> Result>> { if self.computed { return Ok(None); } - self.input.ensure_closed(Reader::Vector(path))?; + self.input.ensure_closed(Reader::Vector)?; let retained = self.input.held; let assembly = self .input .assembly_bytes(&self.schema) - .ok_or_else(|| vector_overflow(path))?; + .ok_or_else(vector_overflow)?; buffer.grow(assembly)?; - let assembled = match self.input.assemble(&self.schema, Reader::Vector(path)) { + let assembled = match self.input.assemble(&self.schema, Reader::Vector) { Ok(assembled) => assembled, Err(error) => { buffer.resize(retained)?; @@ -1718,11 +1613,12 @@ impl VectorRuntime { } /// The error for a vector operation whose reservation does not fit an address space. -fn vector_overflow(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!( - "the input of the vector operation at {path} does not fit one address space: the bytes it would \ +fn vector_overflow() -> DataFusionError { + DataFusionError::Execution( + "the input of the vector operation does not fit one address space: the bytes it would \ reserve overflow" - )) + .to_string(), + ) } /// Runs the kernel `op` binds over the whole input. @@ -2099,81 +1995,412 @@ fn no_row_chunk(schema: &SeriesSchema, metric: MetricName) -> Result, - /// The histogram tables queued results share with each other, charged to the queue as a whole - /// rather than to one result: a table stays alive while any result referencing it is queued, - /// so its charge is given back when the queue empties, not when the first result leaves. - queued_tables: usize, - /// What an instant node holds: the chunks of the series it is selecting, and, while it gathers - /// that series, the workspace the gather reserved for. `None` for every other kind of node. - instant: Option, - /// A window node's evaluator, with its own input and output grid; `None` before it is built and - /// once it is closed. - window: Option, - /// What a map node applies. - map: Option, - /// What a binary node holds when it matches two vectors: both operands, until the input ends. - /// - /// `None` for every other node, and for a binary node that broadcasts a constant instead: that - /// one holds no operand at all and is described by [`NodeRuntime::broadcast`]. - binary: Option, - /// What an aggregate node holds: the fold over what it was handed. - /// - /// Unlike a binary node's operands, nothing of the input survives a chunk: the fold keeps its - /// states and the chunk it read is dropped, so this is the one node whose reservation does not - /// follow what it was handed but what it accumulated. That is also why this node cannot report - /// its input complete from what it holds -- there is nothing to hold -- and reads the driver's - /// [`NativeExecutor::inputs_finished`] and its child's [`NativeExecutionState::settled`] instead. - aggregate: Option, - /// What a vector node holds. - vector: Option, - /// What a binary node holds when one operand is a constant: the operation and the layout of the - /// vector side. - /// - /// The two modes are exclusive. A broadcast transforms each chunk of the vector side as it - /// arrives and retains none of them, so what it holds is its own configuration and not data. - broadcast: Option, - /// The layout the caller's chunks of an input node are checked against. - /// - /// A tree with a binary node reads two inputs, and each of them is bound to its own layout, so - /// the binding is kept per input rather than once for the tree. - source_schema: Option>, - /// What this node has already handed out under each output label set, for a node that rewrites - /// labels: a window drops the metric name and the empty-valued labels, a map drops the name or - /// rewrites a label. - emitted: Option, - /// The input a window collects until its input ends, for a window whose input can hold two - /// series under one label set; `None` for a window that evaluates its chunks as they arrive, - /// and once the flush that merges the collected input has taken it. - merge: Option, - /// Whether the series the last chunk handed over belongs to is still open. - /// - /// Only an input node ever sets this, because its queue is the one the caller fills; the flag is - /// what reports a truncated input to a tree that has no window, since nothing else in such a - /// tree holds a series and so nothing else could notice one that never ended. It is assigned on - /// every handover -- `false` when that chunk carried `is_last`, `true` when it did not -- and - /// cleared when the node is released, so a true flag says exactly one thing: the last chunk the - /// consumer saw did not close its series, and no chunk came after it. Nothing else writes it, so - /// it cannot report a truncation that did not happen. - /// - /// "The series of the last chunk handed over" is the whole answer because no more than one - /// series can be open here: [`NativeExecutionState::try_new`] refuses an input declaring - /// `SeriesOrdering::Interleaved`, so the chunks of a series arrive together and the series last - /// handed over is the only one that could still be open. - open_source_series: bool, +/// The recursive runtime tree shared with observers of one execution. +pub(crate) type RuntimeTreeHandle = Arc>; + +/// The root of one execution's owned runtime topology. +pub(crate) struct RuntimeTree { + root: RuntimeNode, } -impl NodeRuntime { +/// A runtime operator and its directly owned children and metrics. +struct RuntimeNode { + metrics: Arc, + op: RuntimeOp, +} + +enum RuntimeOp { + Input(InputId), + Scalar, + Instant { + input: Box, + instant: Option, + state: NodeState, + }, + Window { + input: Box, + window: Option, + merge: Option, + state: NodeState, + }, + Map { + input: Box, + map: Option, + state: NodeState, + }, + Binary { + left: Box, + right: Box, + binary: Option, + state: NodeState, + }, + Broadcast { + left: Box, + right: Box, + broadcast: Option, + state: NodeState, + }, + Aggregate { + input: Box, + aggregate: Option, + state: NodeState, + }, + Vector { + input: Box, + vector: Option, + state: NodeState, + }, +} + +/// State shared by the stateful operators. +struct NodeState { + queued: VecDeque<(NativeChunk, usize)>, + queued_tables: usize, + emitted: Option, + buffer: NodeBuffer, +} + +/// The caller-owned state of one input leaf. +struct InputSlot { + schema: Arc, + open_source_series: bool, + state: NodeState, +} + +/// State for one execution of a bound tree. +pub struct NativeExecutionState { + runtime: RuntimeTreeHandle, + inputs: Vec, +} + +impl NodeState { + fn new(name: &'static str, metrics: Arc, pool: &Arc) -> Self { + Self { + queued: VecDeque::new(), + queued_tables: 0, + emitted: None, + buffer: NodeBuffer::new(name, metrics, pool), + } + } +} + +impl RuntimeNode { + fn build( + node: &NativeNode, + pool: &Arc, + inputs: &mut Vec, + ) -> Result<(Self, bool)> { + let metrics = Arc::new(NodeMetrics::default()); + let (op, shared) = match &node.expr { + NativeExpr::Input(bound) => { + if bound.ordering == SeriesOrdering::Interleaved { + return Err(DataFusionError::NotImplemented(format!( + "the native runtime does not accept {:?} input ordering", + bound.ordering + ))); + } + let schema = node.output.samples().cloned().ok_or_else(|| { + DataFusionError::Execution("input node does not produce samples".to_string()) + })?; + let id = InputId(inputs.len()); + inputs.push(InputSlot { + schema, + open_source_series: false, + state: NodeState::new("native.input", Arc::clone(&metrics), pool), + }); + (RuntimeOp::Input(id), false) + } + NativeExpr::Scalar(_) => (RuntimeOp::Scalar, false), + NativeExpr::Instant { input, op } => { + let (child, shared) = Self::build(input, pool, inputs)?; + let input_schema = input.output.samples().cloned().ok_or_else(|| { + DataFusionError::Execution( + "instant selection consumes non-sample input".to_string(), + ) + })?; + ( + RuntimeOp::Instant { + input: Box::new(child), + instant: Some(InstantRuntime::try_new(&input_schema, op)?), + state: NodeState::new("native.instant", Arc::clone(&metrics), pool), + }, + shared, + ) + } + NativeExpr::Window { input, op } => { + let (child, child_shared) = Self::build(input, pool, inputs)?; + let input_schema = input.output.samples().cloned().ok_or_else(|| { + DataFusionError::Execution("window consumes non-sample input".to_string()) + })?; + let merge = child_shared.then(|| MergeInput::new(Arc::clone(&input_schema))); + let shared = op.function.drops_metric_name() || !child_shared; + let mut state = NodeState::new("native.window", Arc::clone(&metrics), pool); + state.emitted = Some(Emitted::default()); + ( + RuntimeOp::Window { + input: Box::new(child), + window: Some(RangeEvaluator::try_new( + &input_schema, + Arc::clone(&op.output_grid), + op.window, + op.function, + )?), + merge, + state, + }, + shared, + ) + } + NativeExpr::Map { input, op } => { + let (child, _) = Self::build(input, pool, inputs)?; + let schema = node.output.samples().cloned().ok_or_else(|| { + DataFusionError::Execution("map does not produce samples".to_string()) + })?; + let mut state = NodeState::new("native.map", Arc::clone(&metrics), pool); + state.emitted = Some(Emitted::default()); + ( + RuntimeOp::Map { + input: Box::new(child), + map: Some(MapRuntime { + ops: Arc::clone(&op.ops), + schema, + }), + state, + }, + true, + ) + } + NativeExpr::Binary { + left, + right, + matching, + } => { + let left_schema = left.output.samples(); + let right_schema = right.output.samples(); + let (binary, broadcast) = match (left_schema, right_schema) { + (Some(left_schema), Some(right_schema)) + if left_schema.eval_grid.is_some() && right_schema.eval_grid.is_some() => + { + ( + Some(BinaryRuntime::new( + Arc::new(matching.clone()), + [Arc::clone(left_schema), Arc::clone(right_schema)], + )), + None, + ) + } + _ => (None, Some(broadcast_runtime(left, right, matching)?)), + }; + let (left_node, left_shared) = Self::build(left, pool, inputs)?; + let (right_node, right_shared) = Self::build(right, pool, inputs)?; + match (binary, broadcast) { + (Some(binary), None) => ( + RuntimeOp::Binary { + left: Box::new(left_node), + right: Box::new(right_node), + binary: Some(binary), + state: NodeState::new("native.binary", Arc::clone(&metrics), pool), + }, + false, + ), + (None, Some(broadcast)) => { + let shared = broadcast.drops_labels() + || if left_schema.is_some() { + left_shared + } else { + right_shared + }; + let mut state = NodeState::new("native.binary", Arc::clone(&metrics), pool); + if broadcast.drops_labels() { + state.emitted = Some(Emitted::default()); + } + ( + RuntimeOp::Broadcast { + left: Box::new(left_node), + right: Box::new(right_node), + broadcast: Some(broadcast), + state, + }, + shared, + ) + } + _ => { + return Err(DataFusionError::Internal( + "binary runtime build selected conflicting operation modes".to_string(), + )); + } + } + } + NativeExpr::Aggregate { input, op } => { + let schema = node + .output + .samples() + .or_else(|| input.output.samples()) + .cloned() + .ok_or_else(|| { + DataFusionError::Execution("aggregate does not produce samples".to_string()) + })?; + let mut state = NodeState::new("native.aggregate", Arc::clone(&metrics), pool); + let aggregate = Some(AggregateRuntime::new( + op.clone(), + schema, + &mut state.buffer, + )?); + let (child, _) = Self::build(input, pool, inputs)?; + ( + RuntimeOp::Aggregate { + input: Box::new(child), + aggregate, + state, + }, + false, + ) + } + NativeExpr::Vector { input, op } => { + let schema = node.output.samples().cloned().ok_or_else(|| { + DataFusionError::Execution( + "vector operation does not produce samples".to_string(), + ) + })?; + let one_field = matches!( + op, + BoundVectorOp::TopK { .. } + | BoundVectorOp::LimitK { .. } + | BoundVectorOp::Sort(SortOrder::Asc | SortOrder::Desc) + ); + if one_field && schema.fields.len() != 1 { + return Err(DataFusionError::NotImplemented( + "the native runtime does not execute multi-field vector operations" + .to_string(), + )); + } + let mut vector = VectorRuntime::new(op.clone(), schema); + let (child, child_shared) = Self::build(input, pool, inputs)?; + let limitk = matches!(op, BoundVectorOp::LimitK { .. }); + if limitk && child_shared { + vector.merge_input = true; + } + let whole = limitk + || matches!( + op, + BoundVectorOp::Absent { .. } + | BoundVectorOp::Histogram(_) + | BoundVectorOp::CountValues { .. } + ); + ( + RuntimeOp::Vector { + input: Box::new(child), + vector: Some(vector), + state: NodeState::new("native.vector", Arc::clone(&metrics), pool), + }, + child_shared && !whole, + ) + } + }; + Ok((Self { metrics, op }, shared)) + } + + fn register_metrics(&self, name: &str, set: &ExecutionPlanMetricsSet, partition: usize) { + self.metrics.register(name, set, partition); + match &self.op { + RuntimeOp::Input(_) | RuntimeOp::Scalar => {} + RuntimeOp::Instant { input, .. } + | RuntimeOp::Window { input, .. } + | RuntimeOp::Map { input, .. } + | RuntimeOp::Aggregate { input, .. } + | RuntimeOp::Vector { input, .. } => { + input.register_metrics(&format!("{name}.input"), set, partition) + } + RuntimeOp::Binary { left, right, .. } | RuntimeOp::Broadcast { left, right, .. } => { + left.register_metrics(&format!("{name}.left"), set, partition); + right.register_metrics(&format!("{name}.right"), set, partition); + } + } + } + + fn reserved_bytes(&self) -> usize { + match &self.op { + RuntimeOp::Input(_) | RuntimeOp::Scalar => 0, + RuntimeOp::Instant { input, state, .. } + | RuntimeOp::Window { input, state, .. } + | RuntimeOp::Map { input, state, .. } + | RuntimeOp::Aggregate { input, state, .. } + | RuntimeOp::Vector { input, state, .. } => state + .buffer + .reservation + .size() + .saturating_add(input.reserved_bytes()), + RuntimeOp::Binary { + left, right, state, .. + } + | RuntimeOp::Broadcast { + left, right, state, .. + } => state + .buffer + .reservation + .size() + .saturating_add(left.reserved_bytes()) + .saturating_add(right.reserved_bytes()), + } + } +} + +impl RuntimeTree { + pub(crate) fn export_metrics(&self, partition: usize) -> MetricsSet { + let set = ExecutionPlanMetricsSet::new(); + self.root.register_metrics("root", &set, partition); + set.clone_inner() + } +} + +impl NativeExecutionState { + pub fn try_new(tree: &NativeTree, pool: Arc) -> Result { + let mut inputs = Vec::new(); + let (root, _) = RuntimeNode::build(tree.root(), &pool, &mut inputs)?; + match tree.root().output.samples() { + Some(schema) if schema.eval_grid.is_some() => {} + Some(_) => { + return Err(DataFusionError::Execution( + "native runtime root produces raw samples".to_string(), + )); + } + None => { + return Err(DataFusionError::Execution( + "native runtime root produces no sample output".to_string(), + )); + } + } + if inputs.is_empty() { + return Err(DataFusionError::Execution( + "native tree has no input node".to_string(), + )); + } + let runtime = Arc::new(Mutex::new(RuntimeTree { root })); + Ok(Self { runtime, inputs }) + } + + pub(crate) fn runtime_tree(&self) -> RuntimeTreeHandle { + Arc::clone(&self.runtime) + } + pub fn input_count(&self) -> usize { + self.inputs.len() + } + pub fn reserved_bytes(&self) -> usize { + self.inputs + .iter() + .map(|slot| slot.state.buffer.reservation.size()) + .fold(0usize, usize::saturating_add) + .saturating_add( + self.runtime + .lock() + .unwrap_or_else(|error| error.into_inner()) + .root + .reserved_bytes(), + ) + } +} + +impl NodeState { /// Queues a source chunk, holding what it is worth until it leaves. /// /// What this node holds is its queue and nothing else, so the reservation is the sum of the @@ -2219,31 +2446,11 @@ impl NodeRuntime { Ok(Some(chunk)) } - /// Hands over the next queued source chunk, if there is one. - /// - /// The handover is what [`Self::open_source_series`] follows: after this call the flag says - /// whether the series the consumer just received a chunk of is still open, which is what closing - /// the execution reads to tell a truncated input from one that ended cleanly. Only an input node - /// follows that flag, so only it reads the handover this way: a binary node's queue holds the - /// results of series that already closed on their way in. - fn pop_source(&mut self) -> Result> { - let Some(chunk) = self.pop_queued(NativeChunk::positions)? else { - return Ok(None); - }; - // Assigned rather than only set: a chunk that closes its series is what says no series is - // open any more, so the flag follows the handover and not the history of handovers. - self.open_source_series = !chunk.is_last(); - if chunk.is_last() { - self.buffer.metrics.record_completed_series(1); - } - Ok(Some(chunk)) - } - /// Maps one evaluated chunk, keeping its axis and its closing. /// /// `None` means the operation emptied every position and the chunk does not close its series: /// there is nothing to hand on, and the input may still decide more. - fn map(&mut self, path: &NodePath, chunk: EvaluatedChunk) -> Result> { + fn map(&mut self, map: &MapRuntime, chunk: EvaluatedChunk) -> Result> { let (positions, closing) = (chunk.series.time.len(), chunk.is_last); // The chunk is held for as long as this node works on it. self.buffer.grow(evaluated_bytes(&chunk))?; @@ -2252,16 +2459,9 @@ impl NodeRuntime { // The timer covers this node's own work only: it is dropped before the driver polls // the input again. let _timer = metrics.timer(); - match self.map.as_ref() { - // A label change can write a label far larger than the chunk, so each one is - // admitted before it is built. - Some(map) => map_chunk(&map.schema, &chunk, &map.ops, &mut |bytes| { - self.buffer.grow(bytes) - }), - None => Err(DataFusionError::Execution(format!( - "the node at {path} holds no map" - ))), - } + map_chunk(&map.schema, &chunk, &map.ops, &mut |bytes| { + self.buffer.grow(bytes) + }) }; metrics.record_input(1, positions); if closing { @@ -2275,7 +2475,7 @@ impl NodeRuntime { // What it handed out is what the label sets of this node now hold: `map_chunk` has // already rewritten the output's label set, so the label set of the output is the one // two input series can share. - self.track(path, output)?; + self.track(output)?; } // The chunk is gone and the label sets this node has handed out are what is left. let emitted = self.emitted_bytes(); @@ -2289,11 +2489,15 @@ impl NodeRuntime { /// there is nothing to hand on, and the vector side may still decide more. Nothing of the chunk /// is retained — the operation pairs one value with one constant — so what this node holds /// afterwards is the label sets it has handed out and nothing else. - fn broadcast(&mut self, path: &NodePath, chunk: NativeChunk) -> Result> { + fn broadcast( + &mut self, + broadcast: &BroadcastRuntime, + chunk: NativeChunk, + ) -> Result> { let NativeChunk::Evaluated(chunk) = chunk else { - return Err(DataFusionError::Execution(format!( - "the binary node at {path} broadcasts over an evaluated vector, not raw samples" - ))); + return Err(DataFusionError::Execution( + "the binary node broadcasts over an evaluated vector, not raw samples".to_string(), + )); }; let (positions, closing) = (chunk.series.time.len(), chunk.is_last); // The chunk is held for as long as this node works on it. @@ -2303,10 +2507,7 @@ impl NodeRuntime { // The timer covers this node's own work only: it is dropped before the driver polls // the vector side again. let _timer = metrics.timer(); - match self.broadcast.as_ref() { - Some(broadcast) => broadcast.apply(&chunk), - None => Err(no_broadcast(path)), - } + broadcast.apply(&chunk) }; metrics.record_input(1, positions); if closing { @@ -2327,12 +2528,8 @@ impl NodeRuntime { // one, and only that one keeps the table `track` records into. A filtering comparison // keeps the label set it read, so recording it would refuse two input series that // share it without this node having made them share it. - if self - .broadcast - .as_ref() - .is_some_and(BroadcastRuntime::drops_labels) - { - self.track(path, output)?; + if broadcast.drops_labels() { + self.track(output)?; } } // The chunk is gone and the label sets this node has handed out are what is left. @@ -2348,15 +2545,20 @@ impl NodeRuntime { /// has ended, and reading them as separate series would close the state of the one before in /// the evaluator. Every other window evaluates what it is handed as it arrives, which is what /// lets it answer before its input series ends. - fn window(&mut self, path: &NodePath, chunk: NativeChunk) -> Result> { - if self.merge.is_some() { + fn window( + &mut self, + window: &mut RangeEvaluator, + merge: &mut Option, + chunk: NativeChunk, + ) -> Result> { + if let Some(merge) = merge.as_mut() { let NativeChunk::Evaluated(chunk) = chunk else { - return Err(DataFusionError::Execution(format!( - "the window at {path} collects evaluated fragments, but it was handed raw \ - samples" - ))); + return Err(DataFusionError::Execution( + "the window collects evaluated fragments, but it was handed raw samples" + .to_string(), + )); }; - return self.collect(path, chunk); + return self.collect(merge, window, chunk); } let (positions, closing) = (chunk.positions(), chunk.is_last()); // The chunk is held for as long as the evaluator consumes it. @@ -2364,18 +2566,15 @@ impl NodeRuntime { let metrics = Arc::clone(&self.buffer.metrics); let outcome = { let _timer = metrics.timer(); - match (self.window.as_mut(), &chunk) { + match &chunk { // Raw samples are read window by window by the evaluator itself. - (Some(evaluator), NativeChunk::Raw(chunk)) => evaluator.push(chunk), + NativeChunk::Raw(chunk) => window.push(chunk), // An evaluated input is read as computed samples: the evaluator carries the window // state of one series across its chunks and closes it on the chunk that says so. // Handing each chunk over as it arrives is what lets this window report a decision // before its input series has ended; buffering the chunks into a whole series first // would hold the input's bytes and delay every result to the end of the series. - (Some(evaluator), NativeChunk::Evaluated(chunk)) => { - evaluator.push_series(&chunk.series, chunk.is_last) - } - (None, _) => Err(closed(path)), + NativeChunk::Evaluated(chunk) => window.push_series(&chunk.series, chunk.is_last), } }; metrics.record_input(1, positions); @@ -2389,7 +2588,7 @@ impl NodeRuntime { if let Some(output) = &output { metrics.record_grid_points(output.series.time.len()); metrics.record_output(1, valid_values(output)); - self.track(path, output)?; + self.track(output)?; } // The chunk has been consumed: what this node still holds is what the evaluator kept of it // and the label sets it has already handed out. Both are charged whether or not this chunk @@ -2397,10 +2596,7 @@ impl NodeRuntime { // earlier series' history exactly where they were -- and neither is charged per chunk // consumed, which would make a fixed window over a long series read as an allocation // failure even though the window holds no more samples. - let retained = self - .window - .as_ref() - .map_or(0, RangeEvaluator::retained_bytes); + let retained = window.retained_bytes(); let emitted = self.emitted_bytes(); self.buffer.resize(retained.saturating_add(emitted))?; Ok(output) @@ -2417,26 +2613,26 @@ impl NodeRuntime { /// The chunk is reserved for before it is retained, by the figure the operand reports for it, so /// the pool admits the memory before the node holds it and a refused chunk leaves the operand /// where it was. - fn retain_binary(&mut self, path: &NodePath, side: usize, chunk: NativeChunk) -> Result<()> { + fn retain_binary( + &mut self, + binary: &mut BinaryRuntime, + side: usize, + chunk: NativeChunk, + ) -> Result<()> { let NativeChunk::Evaluated(chunk) = chunk else { - return Err(DataFusionError::Execution(format!( - "the binary node at {path} consumes evaluated chunks, not raw samples" - ))); + return Err(DataFusionError::Execution( + "the binary node consumes evaluated chunks, not raw samples".to_string(), + )); }; // The figure is read before the chunk is retained, because reserving after the fact would be // reserving for memory the node already holds. - let pending = self - .binary - .as_ref() - .ok_or_else(|| no_binary(path))? - .pending(path, side, &chunk)?; + let pending = binary.pending(side, &chunk)?; let positions = chunk.series.time.len(); self.buffer.grow(pending)?; let metrics = Arc::clone(&self.buffer.metrics); { let _timer = metrics.timer(); - let binary = self.binary.as_mut().ok_or_else(|| no_binary(path))?; - binary.retain(path, side, chunk)?; + binary.retain(side, chunk)?; let retained = binary.retained_bytes(); self.buffer.resize(retained)?; } @@ -2445,11 +2641,8 @@ impl NodeRuntime { } /// Runs this node's binary kernel and queues the chunks it produced. - fn compute_binary(&mut self, path: &NodePath) -> Result<()> { - let output = { - let binary = self.binary.as_mut().ok_or_else(|| no_binary(path))?; - binary.compute(path, &mut self.buffer)? - }; + fn compute_binary(&mut self, binary: &mut BinaryRuntime) -> Result<()> { + let output = binary.compute(&mut self.buffer)?; // A kernel that already ran has moved its result into the queue, which the handover below // charges for: leaving the reservation alone here is what keeps a queued result covered. let Some(output) = output else { @@ -2460,19 +2653,18 @@ impl NodeRuntime { /// Retains one chunk of a vector node's input, reserved for before it is retained, as /// [`Self::retain_binary`] retains an operand's. - fn retain_vector(&mut self, path: &NodePath, chunk: NativeChunk) -> Result<()> { + fn retain_vector(&mut self, vector: &mut VectorRuntime, chunk: NativeChunk) -> Result<()> { let NativeChunk::Evaluated(chunk) = chunk else { - return Err(DataFusionError::Execution(format!( - "the vector operation at {path} consumes evaluated chunks, not raw samples" - ))); + return Err(DataFusionError::Execution( + "the vector operation consumes evaluated chunks, not raw samples".to_string(), + )); }; - let vector = self.vector.as_mut().ok_or_else(|| no_vector(path))?; // What a bound tree says the input produces is not proof of what a chunk carries, so the // chunk is checked against the layout before it is retained. validate_binary_operand(vector.schema.as_ref(), std::slice::from_ref(&chunk.series)) .map_err(|error| { DataFusionError::Execution(format!( - "the vector operation at {path} rejected its input: {error}" + "the vector operation rejected its input: {error}" )) })?; let pending = vector.input.pending(&chunk); @@ -2481,8 +2673,7 @@ impl NodeRuntime { let metrics = Arc::clone(&self.buffer.metrics); { let _timer = metrics.timer(); - let vector = self.vector.as_mut().ok_or_else(|| no_vector(path))?; - vector.input.retain(chunk, Reader::Vector(path))?; + vector.input.retain(chunk, Reader::Vector)?; let retained = vector.input.held; self.buffer.resize(retained)?; } @@ -2491,12 +2682,9 @@ impl NodeRuntime { } /// Runs this node's vector kernel and queues the series it kept. - fn compute_vector(&mut self, path: &NodePath) -> Result<()> { - let (output, shares_input) = { - let vector = self.vector.as_mut().ok_or_else(|| no_vector(path))?; - let shares_input = matches!(vector.op, BoundVectorOp::Sort(_)); - (vector.compute(path, &mut self.buffer)?, shares_input) - }; + fn compute_vector(&mut self, vector: &mut VectorRuntime) -> Result<()> { + let shares_input = matches!(vector.op, BoundVectorOp::Sort(_)); + let output = vector.compute(&mut self.buffer)?; let Some(output) = output else { return Ok(()); }; @@ -2510,7 +2698,7 @@ impl NodeRuntime { /// field there is one slice per input chunk of buffers and histogram tables the chunks shared, /// so each series is measured the way the input was ([`operand_chunk_bytes`]) rather than by /// the allocations its slices belong to, which would count a buffer once per chunk. The tables - /// are billed once, to the queue ([`NodeRuntime::queued_tables`]), since several results can + /// are billed once, to the queue ([`NodeState::queued_tables`]), since several results can /// keep one alive. fn queue_results(&mut self, output: Vec, shares_input: bool) -> Result<()> { let mut tables = HashSet::new(); @@ -2555,29 +2743,25 @@ impl NodeRuntime { /// reservation settles afterwards on the fold that is left. Unlike [`Self::retain_binary`], what /// stops being held here is the chunk itself: the fold keeps its states, so a chunk that closes /// its series leaves nothing behind and the node is empty of input again. - fn retain_aggregate(&mut self, path: &NodePath, chunk: NativeChunk) -> Result<()> { + fn retain_aggregate( + &mut self, + aggregate: &mut AggregateRuntime, + chunk: NativeChunk, + ) -> Result<()> { let NativeChunk::Evaluated(chunk) = chunk else { - return Err(DataFusionError::Execution(format!( - "the aggregate at {path} consumes evaluated chunks, not raw samples" - ))); + return Err(DataFusionError::Execution( + "the aggregate consumes evaluated chunks, not raw samples".to_string(), + )); }; // The figure is read before the chunk is folded, because reserving after the fact would be // reserving for memory the fold already read. - let pending = self - .aggregate - .as_ref() - .ok_or_else(|| no_aggregate(path))? - .pending(&chunk); + let pending = aggregate.pending(&chunk); let positions = chunk.series.time.len(); self.buffer.grow(pending)?; let metrics = Arc::clone(&self.buffer.metrics); { let _timer = metrics.timer(); - let (aggregate, buffer) = (&mut self.aggregate, &mut self.buffer); - aggregate - .as_mut() - .ok_or_else(|| no_aggregate(path))? - .accumulate(path, chunk, buffer, pending)?; + aggregate.accumulate(chunk, &mut self.buffer, pending)?; } metrics.record_input(1, positions); // A chunk that closed its series closed it for the node below: a group is not a series, so @@ -2591,13 +2775,10 @@ impl NodeRuntime { /// Each group is one series, decided in full by the fold, so the one chunk it is handed over as /// closes it: a consumer above reads "nothing more here" from that chunk rather than from another /// one arriving. - fn compute_aggregate(&mut self, path: &NodePath) -> Result<()> { + fn compute_aggregate(&mut self, aggregate: &mut AggregateRuntime) -> Result<()> { let metrics = Arc::clone(&self.buffer.metrics); let _timer = metrics.timer(); - let groups = self - .aggregate - .as_ref() - .map_or(0, |aggregate| aggregate.groups()); + let groups = aggregate.groups(); let queue_capacity = self.queued.capacity(); let queue_target = self.queued.len().saturating_add(groups); let queue_growth = queue_target @@ -2614,10 +2795,7 @@ impl NodeRuntime { let queue_bytes = queue_target .max(queue_capacity) .saturating_mul(std::mem::size_of::<(NativeChunk, usize)>()); - let output = { - let aggregate = self.aggregate.as_mut().ok_or_else(|| no_aggregate(path))?; - aggregate.compute(path, &mut self.buffer, queue_bytes)? - }; + let output = aggregate.compute(&mut self.buffer, queue_bytes)?; // The fold has returned its series; reserve their handover before appending them to the queue. let Some(output) = output else { return Ok(()); @@ -2649,11 +2827,15 @@ impl NodeRuntime { /// /// `None` means the series this chunk belongs to has not ended: the selection reads one whole /// axis, so it decides nothing until the chunk carrying `is_last` arrives. - fn instant(&mut self, path: &NodePath, chunk: NativeChunk) -> Result> { + fn instant( + &mut self, + instant: &mut InstantRuntime, + chunk: NativeChunk, + ) -> Result> { let NativeChunk::Raw(chunk) = chunk else { - return Err(DataFusionError::Execution(format!( - "the instant selection at {path} consumes raw samples, not an evaluated chunk" - ))); + return Err(DataFusionError::Execution( + "the instant selection consumes raw samples, not an evaluated chunk".to_string(), + )); }; let (positions, closing) = (chunk.time.len(), chunk.is_last); // The chunk is held from here on: a selection keeps every chunk of a series until the one @@ -2668,11 +2850,6 @@ impl NodeRuntime { // and a selection over a million grid points would take its tens of megabytes without // asking. [`InstantRuntime::gather_bytes`] derives the figure, because the chunks it counts // are that node's. - let Some(instant) = self.instant.as_ref() else { - return Err(DataFusionError::Execution(format!( - "the node at {path} holds no instant selection" - ))); - }; let scratch = if closing { instant.gather_bytes(&chunk) } else { @@ -2684,14 +2861,7 @@ impl NodeRuntime { // The timer covers this node's own work only: it is dropped before the driver polls // the input again. let _timer = metrics.timer(); - match self.instant.as_mut() { - Some(instant) => instant.push(path, chunk), - // Rejected when the state was built, since a selection reads raw samples; kept as - // an error rather than a panic so an impossible tree cannot take the process down. - None => Err(DataFusionError::Execution(format!( - "the node at {path} holds no instant selection" - ))), - } + instant.push(chunk) }; metrics.record_input(1, positions); if closing { @@ -2706,7 +2876,7 @@ impl NodeRuntime { let output = match outcome { Ok(output) => output, Err(error) => { - self.settle_buffered()?; + self.settle_buffered(instant)?; return Err(error); } }; @@ -2718,7 +2888,7 @@ impl NodeRuntime { // held until it ends, so the reservation is what those chunks hold rather than a figure // that grows with the chunk count of anything already answered. The chunk just consumed and // the workspace of the gather it closed are both gone, so this is what gives them back. - self.settle_buffered()?; + self.settle_buffered(instant)?; Ok(output) } @@ -2726,12 +2896,8 @@ impl NodeRuntime { /// /// The chunk the node just consumed and the workspace of a gather it closed are not part of it: /// the first has left the buffer and the second has been handed to the output. - fn settle_buffered(&mut self) -> Result<()> { - let buffered = self - .instant - .as_ref() - .map_or(0, InstantRuntime::buffered_bytes); - self.buffer.resize(buffered) + fn settle_buffered(&mut self, instant: &InstantRuntime) -> Result<()> { + self.buffer.resize(instant.buffered_bytes()) } /// Keeps one fragment of this node's collected input, charged for as it arrives. @@ -2743,18 +2909,14 @@ impl NodeRuntime { /// separately before it runs. fn collect( &mut self, - path: &NodePath, + merge: &mut MergeInput, + window: &RangeEvaluator, chunk: EvaluatedChunk, ) -> Result> { - let Some(merge) = self.merge.as_mut() else { - return Err(DataFusionError::Execution(format!( - "the window at {path} holds no collected input" - ))); - }; let EvaluatedChunk { series, is_last } = chunk; // The chunk is checked before it is charged for or kept, so a stream that breaks the // contract leaves nothing behind. - merge.check(path, &series, is_last)?; + merge.check(&series, is_last)?; let positions = series.time.len(); let bytes = series_bytes(&series); // The fragment is held from here on, so the pool admits it before this node keeps it: a @@ -2771,17 +2933,86 @@ impl NodeRuntime { } // What this node holds is the fragments and the label sets it has already handed out; its // evaluator has read none of them yet. - let retained = self - .window - .as_ref() - .map_or(0, RangeEvaluator::retained_bytes); + let retained = window.retained_bytes(); let emitted = self.emitted_bytes(); - let pending = self.merge.as_ref().map_or(0, |merge| merge.bytes); self.buffer - .resize(retained.saturating_add(emitted).saturating_add(pending))?; + .resize(retained.saturating_add(emitted).saturating_add(merge.bytes))?; Ok(None) } + /// Merges this window's collected input and evaluates it, one merged series at a time. + /// + /// Each merged series is a whole series: it is handed to the evaluator as one closing series, so + /// the window reports on it once, and its chunk waits in this node's queue at its own estimate. + /// The fragments and the merge's workspace stay charged for the whole flush and go back when it + /// ends; the queued answers stay charged until the pull that hands each of them up. + /// + /// `merge` is taken here, so a window whose collected input failed to merge does not merge it + /// again: the error has stopped the execution, and the fragments went with it. + fn flush_merge( + &mut self, + window: &mut RangeEvaluator, + merge: &mut Option, + ) -> Result<()> { + let Some(merge) = merge.take() else { + return Ok(()); + }; + if merge.fragments.is_empty() { + // Nothing arrived, so there is nothing to merge and nothing to answer: the window's + // evaluator never read a series and the charge goes back. + return self.buffer.shrink(merge.bytes); + } + // The workspace the merge allocates is charged before it runs, from the rows, segments and + // fields of the fragments as they were collected. + let workspace = merge.workspace_bytes(); + let held = merge.bytes.saturating_add(workspace); + let schema = Arc::clone(&merge.schema); + self.buffer.grow(workspace)?; + // What the queue holds before this flush adds to it: a queued answer is charged until the + // pull that hands it up returns its estimate, so every resize below has to leave it charged. + // The figure is folded forward rather than read back off the queue, because walking the + // queue once per answer would make a flush cost a pass over what it produced so far. + let mut queued_bytes = self + .queued + .iter() + .fold(self.queued_tables, |total, (_, bytes)| { + total.saturating_add(*bytes) + }); + // The fragments' charge stands for the merged series too -- a merged run is a slice of a + // fragment's own segment, so no value is copied -- and the workspace stays charged until + // the flush is over. + let merged = merge_same_labelset(&schema, merge.fragments)?; + for series in merged { + let output = { + let metrics = Arc::clone(&self.buffer.metrics); + let _timer = metrics.timer(); + window.push_series(&series, true)? + }; + let Some(output) = output else { + continue; + }; + self.buffer + .metrics + .record_grid_points(output.series.time.len()); + self.buffer.metrics.record_output(1, valid_values(&output)); + self.track(&output)?; + let retained = window.retained_bytes(); + let emitted = self.emitted_bytes(); + self.buffer.resize( + retained + .saturating_add(emitted) + .saturating_add(held) + .saturating_add(queued_bytes), + )?; + let bytes = evaluated_bytes(&output); + self.enqueue_output(output)?; + queued_bytes = queued_bytes.saturating_add(bytes); + } + // The fragments and the merge's workspace go back, and whatever the flush produced stays + // charged in the node's queue until the caller pulls it. + self.buffer.shrink(held) + } + /// Queues one output this node computed while flushing its collected input. /// /// The chunk waits for the pull that hands it up the tree, and its estimate is added as it is @@ -2796,8 +3027,8 @@ impl NodeRuntime { /// Hands over the next output this node computed while flushing, if there is one. /// - /// This is not a source handover: [`Self::open_source_series`] and the input metrics follow the - /// chunks the caller queued, and a chunk this node produced itself is neither. + /// This is not a source handover: an input slot's open-series flag and the input metrics follow + /// the chunks the caller queued, and a chunk this node produced itself is neither. fn take_output(&mut self) -> Result> { let Some((chunk, bytes)) = self.queued.pop_front() else { return Ok(None); @@ -2822,9 +3053,9 @@ impl NodeRuntime { /// where a second one is refused: a node above reads what this one hands over, and a window /// above would evaluate the two as two series and report both. Checking at the root would be /// too late for that — the answer would already have been computed. - fn track(&mut self, path: &NodePath, output: &EvaluatedChunk) -> Result<()> { + fn track(&mut self, output: &EvaluatedChunk) -> Result<()> { match self.emitted.as_mut() { - Some(emitted) => emitted.occupy(path, output), + Some(emitted) => emitted.occupy(output), // A node that keeps its input's labels cannot make two series one. None => Ok(()), } @@ -2837,27 +3068,16 @@ impl NodeRuntime { self.emitted.as_ref().map_or(0, Emitted::bytes) } - /// Returns everything this node holds, including queued chunks. + /// Returns the shared state this node holds: its queue, its tables and its reservation. + /// + /// The operation's own payload — the evaluator, the retained operands, the fold — is not this + /// state's: it belongs to the node that owns this state, and it is dropped there before this + /// runs. Dropping it first is what keeps the reservation from going back to the pool while the + /// memory it was taken for is still held. fn release(&mut self) { - // A node's own state goes first: dropping the evaluator and the buffered series is what - // gives back the memory their reservation was taken for. Returning the reservation while - // they still held the samples would let the pool admit memory the process has not released, - // for as long as the caller keeps the stopped executor alive. The label sets this node - // handed out and the fragments it collected go with it. - self.window = None; - self.instant = None; - self.binary = None; - // The fold is what an aggregate node holds, and dropping it is what gives its states back. - self.aggregate = None; - self.vector = None; - self.broadcast = None; - self.source_schema = None; + // The label sets this node handed out and the chunks it queued go first, for the same + // reason: their memory is what the reservation below returns. self.emitted = None; - self.merge = None; - // A node that holds no chunk holds no series either, so nothing it handed over is still - // open for it: the flag is what closing the execution reads, and it must not survive the - // state that could have kept a series alive. - self.open_source_series = false; self.queued.clear(); self.queued.shrink_to_fit(); self.queued_tables = 0; @@ -2866,615 +3086,59 @@ impl NodeRuntime { } } -/// The run-time state of one execution of a bound tree. -/// -/// The state is keyed by tree position: the same positions [`crate::native::metrics::walk`] -/// derives, so a node's state, its metrics and the name it is shown under cannot drift apart. -/// Because none of it lives on the tree, one compiled tree serves every execution that builds a -/// state from it. -pub struct NativeExecutionState { - /// The tree's structure, flattened in traversal order, so the root is at index 0. - nodes: Vec, - /// The position of each node, in the same order. - paths: Vec, - /// The per-node state, in the same order. - runtimes: Vec, - /// The index of every node, so a caller names a node the way the metrics do. - by_path: HashMap, - /// The index of every input node, in traversal order: where a caller's chunks enter. +impl InputSlot { + /// Queues one source chunk at this input, checking it against the binding the leaf was built + /// with. /// - /// A tree of unary nodes has one, and a binary node adds one per operand: a tree of nested - /// binary nodes has as many as it has leaves, and a queued chunk names the one it belongs to. - sources: Vec, -} - -impl NativeExecutionState { - /// Builds the state of one execution of `tree`. - /// - /// Every node registers its metrics in `set` under its position and partition, and every node - /// that buffers input gets its own reservation from `pool`. - pub fn try_new( - tree: &NativeTree, - pool: Arc, - metrics_set: &ExecutionPlanMetricsSet, - partition: usize, - ) -> Result { - // One pass over the tree names and registers every node in the order - // `crate::native::metrics::walk` gives it: the same rule `EXPLAIN` uses, so a node cannot be - // shown at one position and registered at another. That order reaches a node's left operand - // before its right one, so the input a spec names -- the one input of a unary node, the left - // operand of a binary node -- is the node the pass reaches next. - let mut nodes: Vec = Vec::new(); - let mut paths: Vec = Vec::new(); - let mut runtimes: Vec = Vec::new(); - // The bound expression of each node, in the same order: the pass below reads the node it - // configures from here, so nothing about an operation is copied onto the node's spec. - let mut exprs: Vec<&NativeExpr> = Vec::new(); - // The nodes the pass has not reached yet, the next one last: a node pushes its inputs back - // to front, so the one popped next is the one its parent's spec names. - let mut pending: Vec<(&NativeNode, NodePath)> = vec![(tree.root(), NodePath::root())]; - while let Some((node, path)) = pending.pop() { - let index = nodes.len(); - let handles = Arc::new(NodeMetrics::default()); - handles.register(&path, metrics_set, partition); - let tag = match &node.expr { - NativeExpr::Input(bound) => { - // Every node here reads one series at a time and assumes its chunks arrive - // together: a window feeds a series to its evaluator up to the chunk carrying - // `is_last`, and an input node keeps one flag for the series it handed over, - // which is what reports one that never closed. An input that declares - // `Interleaved` can leave a series open across another series' chunks, which - // either of those would misread, so the tree is refused here rather than run. - // That is a precondition of the runtime, not a limitation to document: the tree - // stays free to declare the ordering -- the compiler's job is not narrowed here - // -- and no tree this runtime executes declares it. - if bound.ordering == SeriesOrdering::Interleaved { - return Err(unsupported_ordering(bound.ordering, &path)); - } - NodeTag::Input - } - NativeExpr::Instant { .. } => NodeTag::Instant, - NativeExpr::Window { .. } => NodeTag::Window, - NativeExpr::Map { .. } => NodeTag::Map, - // A binary node is built here and narrowed below: legality is a wider set of operand - // types than the kernels take, so which of its two modes it is -- a match between - // two vectors, or a constant broadcast over one -- is decided where the operands' - // layouts are read. - NativeExpr::Binary { .. } => NodeTag::Binary, - // An aggregate is built here and narrowed below, for the same reason: `validate` - // admits every kind and every stage as a legal tree, and the runtime computes one - // kind at one stage. - NativeExpr::Aggregate { .. } => NodeTag::Aggregate, - NativeExpr::Vector { .. } => NodeTag::Vector, - // A scalar is a value node: it produces the constant it was bound to, hands over no - // chunk and holds no state, so it is built like any other node and pulled like none. - NativeExpr::Scalar(_) => NodeTag::Scalar, - }; - let buffer = NodeBuffer::new(&path, handles, &pool); - let mut runtime = NodeRuntime { - buffer, - queued: VecDeque::new(), - queued_tables: 0, - instant: None, - window: None, - map: None, - binary: None, - aggregate: None, - vector: None, - broadcast: None, - source_schema: None, - emitted: None, - merge: None, - open_source_series: false, - }; - match &node.expr { - NativeExpr::Input(_) => { - // What the caller's chunks of this input are checked against: a raw source - // carries timestamps, an evaluated one is addressed on a grid. - runtime.source_schema = node.output.samples().cloned(); - } - NativeExpr::Instant { input, op } => { - let input_schema = input.output.samples().ok_or_else(|| { - DataFusionError::Execution(format!( - "the instant selection at {path} consumes samples, but its input \ - produces none" - )) - })?; - runtime.instant = Some(InstantRuntime::try_new(input_schema, op)?); - // A selection keeps the labels of the series it reads, empty-valued ones - // included, so it is not one of the nodes that can turn two series into one - // label set and it keeps no table of what it handed out: the node that drops a - // label is the one that can collide, and it records its own. - } - NativeExpr::Map { op, .. } => { - let schema = node.output.samples().cloned().ok_or_else(|| { - DataFusionError::Execution(format!( - "the map node at {path} does not produce samples" - )) - })?; - runtime.map = Some(MapRuntime { - ops: Arc::clone(&op.ops), - schema, - }); - // A map drops the metric name or rewrites a label, so it is one of the nodes - // that can turn two series into one label set. - runtime.emitted = Some(Emitted::default()); - } - NativeExpr::Window { input, op } => { - let input_schema = input.output.samples().ok_or_else(|| { - DataFusionError::Execution(format!( - "the window at {path} consumes samples, but its input produces none" - )) - })?; - // One evaluator per window, with the grids the node was bound to: the output - // grid is the node's own grid, which is also the grid its chunks are addressed - // on. - runtime.window = Some(RangeEvaluator::try_new( - input_schema, - Arc::clone(&op.output_grid), - op.window, - op.function, - )?); - // A window drops the metric name and the empty-valued labels, so a `rate` over - // a series whose tag is empty and one where it is absent produces one label set - // from two series. - runtime.emitted = Some(Emitted::default()); - } - NativeExpr::Binary { - left, - right, - matching, - } => { - // A binary node has two modes here, and the operands' layouts are what tell - // them apart. Two evaluated vectors are matched by label set and retain both - // operands; one evaluated vector and a constant is a broadcast, which pairs - // each vector value with the constant as the chunk carrying it arrives. All - // that is left is refused as this node, before the pass reaches the operands - // that produced them, so the position named is the one that asked for them. - match (left.output.samples(), right.output.samples()) { - (Some(left), Some(right)) - if left.eval_grid.is_some() && right.eval_grid.is_some() => - { - // Each operand's layout is the one its own child produces, and the walk - // only built this node when both produce evaluated samples. - runtime.binary = Some(BinaryRuntime::new( - Arc::new(matching.clone()), - [Arc::clone(left), Arc::clone(right)], - )); - } - _ => { - let broadcast = broadcast_runtime(&path, left, right, matching)?; - if broadcast.drops_labels() { - // The operation rewrites the label set it hands out, so this node - // is one where two input series can become one: it keeps the table - // that refuses a second value at a position the first one filled. - runtime.emitted = Some(Emitted::default()); - } - runtime.broadcast = Some(broadcast); - } - } - // A binary operation over two vectors matches by label set, so a result label - // set may be produced at points that do not collide; the kernel groups and - // merges them itself, which is why that mode keeps no ledger of what it handed - // out. A broadcast that keeps the labels it reads is not where two series - // become one either, so it keeps none; the one that drops them does, above. - } - NativeExpr::Aggregate { input, op } => { - // A `Complete` aggregate neither adds nor drops a field and evaluates on its - // input's grid, so the node's own output is the layout it folds. What it does - // not execute -- another kind, or another stage -- is refused here, before the - // pass reaches the input that would feed it, and naming what it found is that - // rejection's job: a `Partial` aggregate's output is state and names no samples - // at all, so the input's layout stands in for it and the stage decides. - let schema = node - .output - .samples() - .or_else(|| input.output.samples()) - .cloned() - .ok_or_else(|| { - DataFusionError::Execution(format!( - "the aggregate at {path} does not produce samples" - )) - })?; - runtime.aggregate = Some(AggregateRuntime::new( - op.clone(), - schema, - &mut runtime.buffer, - )?); - } - NativeExpr::Vector { op, .. } => { - let schema = node.output.samples().cloned().ok_or_else(|| { - DataFusionError::Execution(format!( - "the vector operation at {path} does not produce samples" - )) - })?; - // `topk`, `bottomk`, `limitk` and the value sorts read one value per sample, - // so their kernels take one field; the others keep or test whole samples and - // take any number. - let one_field = matches!( - op, - BoundVectorOp::TopK { .. } - | BoundVectorOp::LimitK { .. } - | BoundVectorOp::Sort(SortOrder::Asc | SortOrder::Desc) - ); - if one_field && schema.fields.len() != 1 { - return Err(not_implemented("multi-field Vector", &path)); - } - runtime.vector = Some(VectorRuntime::new(op.clone(), schema)); - // A vector operation keeps the labels of the series it reads, or answers under - // label sets of its own, so it keeps no ledger of what it handed out. - } - NativeExpr::Scalar(_) => { - // A scalar holds nothing: what its consumer reads is the constant the node was - // bound to, which is on the tree rather than in this state. - } - } - // The inputs this node reads, in the order the tree declares them: a unary node's one - // input, a binary node's left and right operand, and none for an input node. They are - // pushed back to front, so the node the spec names is the one the pass reaches next and - // nothing but its own subtree lies between the two. - let reads_input = match &node.expr { - NativeExpr::Input(_) | NativeExpr::Scalar(_) => false, - NativeExpr::Instant { input, .. } - | NativeExpr::Window { input, .. } - | NativeExpr::Map { input, .. } - | NativeExpr::Aggregate { input, .. } - | NativeExpr::Vector { input, .. } => { - pending.push((input.as_ref(), path.input())); - true - } - NativeExpr::Binary { left, right, .. } => { - pending.push((right.as_ref(), path.right())); - pending.push((left.as_ref(), path.left())); - true - } - }; - nodes.push(NodeSpec { - tag, - input: reads_input.then_some(index + 1), - }); - exprs.push(&node.expr); - paths.push(path.clone()); - runtimes.push(runtime); + /// The leaf's own schema says which kind of chunk it produces: a leaf bound to a grid is fed + /// evaluated chunks, and one bound to raw samples is fed raw ones. A chunk of the other kind has + /// no address here, so it is refused rather than queued and read as though it were the kind this + /// leaf promised. + fn enqueue(&mut self, chunk: NativeChunk) -> Result<()> { + let bound = self.schema.eval_grid.is_some(); + let evaluated = matches!(chunk, NativeChunk::Evaluated(_)); + if evaluated != bound { + return Err(DataFusionError::Execution(format!( + "the native source is bound to {}, but {} chunk was queued at it", + if bound { + "an evaluated series" + } else { + "raw samples" + }, + if evaluated { "an evaluated" } else { "a raw" }, + ))); } - - // A caller names a node by the position the metrics give it, and a chunk it queues is bound - // to the input that position names, so every position is indexed once the pass has given the - // whole tree one. The reverse pass below reads it as well, because an operand is not always - // the node the spec's `input` names: a broadcast's vector side may be its right operand. - let by_path: HashMap = paths - .iter() - .enumerate() - .map(|(index, path)| (path.clone(), index)) - .collect(); - - // One reverse pass sets where a merge has to happen, from the input upwards: the pass above - // walked the root first, so a node's input is the node after it and has been decided by the - // time the node itself is. - // - // A source is a stream of series, each with its own label set, so its output needs no merge: - // that is what keeps `Input -> Window` streaming and reporting as its input arrives. A map - // drops the metric name or rewrites a label, so its output can hold two input series under one label set, and - // every node above it needs the merge. A window collects its input and merges it when its - // input can hold two series under one label set: reading them as series one after another - // would end the first in the evaluator before the second arrives, and a window across the - // boundary needs both. - // - // A window's own output needs the same when it rewrites labels -- `rate` drops the name, so - // two input series can become one label set -- or when it did not merge its input: - // `last_over_time` keeps the name, but a tag holding an empty value and an absent tag are - // two input series and one canonical label set, so the pair only becomes one series if it - // was merged before the window read it. - // - // The flags this needs are the pass's own: one per node, filled from the input upwards, and - // they and the expressions go when the pass ends. A node's spec therefore stays what the - // tree said, and nothing about an operation is kept once the state is built. - { - let mut holds_shared: Vec = vec![false; nodes.len()]; - for (index, expr) in exprs.into_iter().enumerate().rev() { - let input_holds_shared = - nodes[index].input.is_some_and(|child| holds_shared[child]); - match nodes[index].tag { - NodeTag::Input => {} - NodeTag::Instant => { - // A selection keeps the labels it read, so it is not where two series - // become one label set and its own output carries its input's flag. Its - // input is the source -- `validate` binds a selection to raw samples, and - // no node above a source produces them -- and a source hands each series' - // chunks over together, so the flag is false there in every tree this - // runtime runs. Should the pass ever compute it true, the selection still - // needs no collected input: it reads one series at a time and refuses a - // chunk of another before it is buffered. - holds_shared[index] = input_holds_shared; - } - NodeTag::Map => holds_shared[index] = true, - NodeTag::Aggregate => { - // An aggregate hands over one series per group, each under the label set its - // group was keyed by and each closed on its own chunk: two groups are two - // label sets, so nothing above has to join pieces of a series. Its own input - // is read whole -- the fold holds what it accumulated, not the chunks -- so - // nothing about its input's flag reaches it either. - holds_shared[index] = false; - } - NodeTag::Scalar => { - // A scalar produces one value and no labels, so it is not a stream of series - // and nothing above it can read two of them under one label set. The flag - // is false rather than inherited: the node has no input at all. - holds_shared[index] = false; - } - NodeTag::Binary => { - // A vector match files one result row per label set and point, merging the - // rows of one set whatever piece they came from: what it hands out is one - // series per label set, so nothing above has to join pieces of a series. Its - // own operands are read whole -- it retains them until the input ends -- so - // its flag is false whatever theirs are. - // - // A broadcast hands each input chunk up transformed instead, so it merges - // nothing: what its output can hold under one label set is what its vector - // side held, plus the pair its own label rewrite can create. One that drops - // the metric name -- arithmetic and a `bool` comparison -- is therefore where - // two input series become one label set, and one that keeps the name only - // inherits the pair its input already had. The flag of the *vector* side is - // the one that matters: the constant is a node with no input at all. - holds_shared[index] = match runtimes[index].broadcast.as_ref() { - Some(broadcast) => { - let vector = match broadcast.operand { - ScalarOperand::Right(_) => nodes[index].input, - ScalarOperand::Left(_) => { - by_path.get(&paths[index].right()).copied() - } - }; - broadcast.drops_labels() - || vector.is_some_and(|vector| holds_shared[vector]) - } - None => false, - }; - } - NodeTag::Vector => { - // Most vector operations keep samples of the series they read, so pieces of one label - // set stay pieces: they hold disjoint points, so ranking them apart picks - // what ranking the merged series would, and what is above treats them as - // it treats a map's. `absent` answers with one series under the labels it - // was bound to, the histogram functions merge the label sets they answer - // with, `count_values` answers with one series per label set it makes, and - // `limitk` counts series in input order, so it merges the pieces first; - // none of them hands over pieces. - let op = match expr { - NativeExpr::Vector { op, .. } => Some(op), - _ => None, - }; - let limitk = matches!(op, Some(BoundVectorOp::LimitK { .. })); - if limitk - && input_holds_shared - && let Some(vector) = runtimes[index].vector.as_mut() - { - vector.merge_input = true; - } - let whole = limitk - || matches!( - op, - Some( - BoundVectorOp::Absent { .. } - | BoundVectorOp::Histogram(_) - | BoundVectorOp::CountValues { .. } - ) - ); - holds_shared[index] = input_holds_shared && !whole; - } - NodeTag::Window => { - let (drops_metric_name, input_schema) = match expr { - NativeExpr::Window { input, op } => ( - op.function.drops_metric_name(), - input.output.samples().cloned(), - ), - _ => { - return Err(DataFusionError::Execution(format!( - "the node at {} is a window without a bound window operation", - paths[index] - ))); - } - }; - if input_holds_shared { - let schema = input_schema.ok_or_else(|| { - DataFusionError::Execution(format!( - "the window at {} merges its input, but its input produces no \ - samples", - paths[index] - )) - })?; - runtimes[index].merge = Some(MergeInput::new(schema)); - } - // A window's own output can hold two series under one label set when it drops - // the name, or when it did not merge its input and so never joined the two. - holds_shared[index] = drops_metric_name || !input_holds_shared; - } - } - } - } - - // The runtime's output is an evaluated chunk. Raw samples have no grid, and a scalar or - // aggregate state is not a query value, so neither can be the tree's answer. - match tree.root().output.samples() { - Some(schema) if schema.eval_grid.is_some() => {} - Some(_) => { - return Err(DataFusionError::Execution( - "the native runtime produces evaluated chunks, but the root of the tree \ - produces raw samples" - .to_string(), - )); - } - None => { - return Err(DataFusionError::Execution( - "the native runtime produces evaluated chunks, but the root of the tree \ - produces a scalar or aggregate state" - .to_string(), - )); - } - } - - // Every input node is where a caller's chunks can enter, and each of them is named by its - // own position when a caller queues a chunk. A tree of unary nodes has one input; a binary - // node adds one per operand, so a tree of nested binary nodes has as many as it has leaves. - let sources: Vec = nodes - .iter() - .enumerate() - .filter(|(_, node)| node.tag == NodeTag::Input) - .map(|(index, _)| index) - .collect(); - if sources.is_empty() { - return Err(DataFusionError::Execution( - "the native tree has no input node".to_string(), - )); - } - // A caller's chunks are samples: a source that hands over aggregate state is not something - // this runtime can be handed, and it is refused when the state is built rather than when the - // first chunk is queued at it. - for &source in &sources { - if runtimes[source].source_schema.is_none() { - return Err(DataFusionError::Execution(format!( - "the input node at {} does not produce samples", - paths[source] - ))); - } - } - Ok(Self { - nodes, - paths, - runtimes, - by_path, - sources, - }) + self.state.enqueue(chunk) } - /// The metrics of the node at `path`. - pub fn metrics(&self, path: &NodePath) -> Option<&Arc> { - let index = self.paths.iter().position(|candidate| candidate == path)?; - Some(&self.runtimes[index].buffer.metrics) - } - - /// The position of the right operand of the binary node at `index`. + /// Hands over the next queued source chunk, if there is one. /// - /// A spec names one input, the node the build pass reaches next: a unary node's only input and a - /// binary node's left operand. The right operand is the one input a spec cannot name, so it is - /// read from the position the node itself sits at, which the pass resolved and indexed. - fn right_input(&self, index: usize) -> Result { - let path = &self.paths[index]; - self.by_path.get(&path.right()).copied().ok_or_else(|| { - DataFusionError::Execution(format!( - "the native tree has no node at the right input of {path}" - )) - }) - } - - /// The bytes this execution currently reserves from the pool. - pub fn reserved_bytes(&self) -> usize { - self.runtimes - .iter() - .map(|runtime| runtime.buffer.reservation.size()) - .sum() - } - - /// How many nodes still hold the state a window buffers into. - /// - /// Test-only: releasing a stopped execution has to drop the evaluators and their sample - /// buffers, not only the numbers that say how many bytes they were reserved for, and that is - /// not observable from the pool. - #[cfg(test)] - pub(crate) fn nodes_holding_windows(&self) -> usize { - self.runtimes - .iter() - .filter(|runtime| runtime.window.is_some()) - .count() - } - - /// How many nodes still hold the chunks of a series an instant selection was gathering. - /// - /// Test-only, and for the same reason as [`Self::nodes_holding_windows`]: the buffered chunks - /// have to be dropped, not only the bytes they were reserved for, and that is not observable - /// from the pool. - #[cfg(test)] - pub(crate) fn nodes_holding_instants(&self) -> usize { - self.runtimes - .iter() - .filter(|runtime| runtime.instant.is_some()) - .count() - } - - /// How many nodes still hold the operands a binary operation retained. - /// - /// Test-only, and for the same reason as [`Self::nodes_holding_windows`]: an execution that - /// stopped has to drop the vectors it retained, not only the number that says how many bytes - /// they were reserved for. - #[cfg(test)] - pub(crate) fn nodes_holding_binary(&self) -> usize { - self.runtimes - .iter() - .filter(|runtime| runtime.binary.is_some()) - .count() - } - - /// The input that ended before its last chunk, if there is one. - /// - /// It is read before anything is merged: a window that collects its input closes every series - /// it merges with a synthetic `is_last`, so a series the caller never ended would be - /// indistinguishable from one that ended afterwards. - fn truncated(&self) -> Option { - self.paths - .iter() - .zip(self.runtimes.iter()) - .find(|(_, runtime)| runtime.open_source_series) - .map(|(path, _)| truncated_input(path)) - } - - /// Reports a stream that ended inside a series, without closing any node. - /// - /// Closing is destructive -- a window's evaluator takes itself apart -- and the binary and - /// vector nodes still have to compute after this check, so what is read here is only the - /// state that follows what each node was handed: an input node reports the series it handed - /// over without the chunk that ends it, and a binary or vector node reports an input that is - /// still open. A truncation is therefore refused before a kernel allocates its workspace, rather - /// than after one has already been computed on half a vector. - fn unclosed(&self) -> Result<()> { - // A source series that never ended is refused before anything is merged: the close a merge - // synthesises would make it look like one that did. - if let Some(error) = self.truncated() { - return Err(error); + /// The handover is what [`Self::open_source_series`] follows: after this call the flag says + /// whether the series the consumer just received a chunk of is still open, which is what closing + /// the execution reads to tell a truncated input from one that ended cleanly. Only an input leaf + /// follows that flag, so only it reads the handover this way: any other node's queue holds the + /// results of series that already closed on their way in. + fn pop_source(&mut self) -> Result> { + let Some(chunk) = self.state.pop_queued(NativeChunk::positions)? else { + return Ok(None); + }; + // Assigned rather than only set: a chunk that closes its series is what says no series is + // open any more, so the flag follows the handover and not the history of handovers. + self.open_source_series = !chunk.is_last(); + if chunk.is_last() { + self.state.buffer.metrics.record_completed_series(1); } - for (path, runtime) in self.paths.iter().zip(&self.runtimes) { - if let Some(binary) = &runtime.binary { - binary.closed(path)?; - } - if let Some(vector) = &runtime.vector { - vector.input.ensure_closed(Reader::Vector(path))?; - } - } - Ok(()) + Ok(Some(chunk)) } - /// Closes every node, reporting an input that ended mid-series. + /// Returns what this input still holds: its open-series flag, its queue and its reservation. /// - /// A drain finds nothing left to emit: a window reports its results on the chunk that closes - /// the series, so what is left to do is to check that every series did close, and to return - /// everything the execution reserved. A window's evaluator is what holds a series and so - /// notices one that never ended; a tree with no window has none, so there the input node answers - /// the same question from [`NodeRuntime::open_source_series`], which follows what it handed over. - /// The first error found is the one reported, and the nodes are walked from the root down, so a - /// window above the input reports its own truncation as it always did. - fn close(&mut self) -> Result<()> { - let mut result = Ok(()); - for (path, runtime) in self.paths.iter().zip(self.runtimes.iter_mut()) { - if let Some(window) = runtime.window.take() - && let Err(error) = window.finish() - && result.is_ok() - { - result = Err(error); - } - if runtime.open_source_series && result.is_ok() { - result = Err(truncated_input(path)); - } - runtime.release(); - } - result + /// A leaf that holds no chunk holds no series either, so nothing it handed over is still open + /// for it: the flag is what closing the execution reads, and it must not survive the state that + /// could have kept a series alive. + fn release(&mut self) { + self.open_source_series = false; + self.state.release(); } } @@ -3535,31 +3199,19 @@ impl NativeExecutor { &self.state } - /// Queues one raw source chunk at the tree's only input. + /// Queues one raw source chunk at the input `input` names. /// - /// A tree with a binary node reads two inputs, and a chunk queued at "the input" would be one - /// the caller cannot say the source of: such a tree is refused here rather than answered from a - /// source the caller did not name, and [`Self::push_raw_at`] is what names one. - pub fn push_raw(&mut self, chunk: RawSeriesChunk) -> Result<()> { - self.push_source(None, NativeChunk::Raw(chunk)) + /// A tree of unary nodes has one input; a binary node adds one per operand, so a tree of nested + /// binary nodes has as many as it has leaves. The id is what says which leaf a chunk belongs to, + /// and a chunk queued at an id the tree has no input for is refused rather than assigned to + /// whichever input was found first. + pub fn push_raw(&mut self, input: InputId, chunk: RawSeriesChunk) -> Result<()> { + self.push_source(input, NativeChunk::Raw(chunk)) } - /// Queues one evaluated source chunk at the tree's only input. - pub fn push_evaluated(&mut self, chunk: EvaluatedChunk) -> Result<()> { - self.push_source(None, NativeChunk::Evaluated(chunk)) - } - - /// Queues one raw source chunk at the input node at `path`. - /// - /// The path is the position the metrics and `EXPLAIN` name the node by, so a caller that reads - /// either can say which of the two inputs a chunk belongs to. - pub fn push_raw_at(&mut self, path: &NodePath, chunk: RawSeriesChunk) -> Result<()> { - self.push_source(Some(path), NativeChunk::Raw(chunk)) - } - - /// Queues one evaluated source chunk at the input node at `path`. - pub fn push_evaluated_at(&mut self, path: &NodePath, chunk: EvaluatedChunk) -> Result<()> { - self.push_source(Some(path), NativeChunk::Evaluated(chunk)) + /// Queues one evaluated source chunk at the input `input` names. + pub fn push_evaluated(&mut self, input: InputId, chunk: EvaluatedChunk) -> Result<()> { + self.push_source(input, NativeChunk::Evaluated(chunk)) } /// The next chunk the root produces, or `None` when nothing has settled yet. @@ -3807,80 +3459,6 @@ impl NativeExecutor { } } - /// Merges the collected input of the node at `index` and evaluates it, one merged series at a - /// time. - /// - /// Each merged series is a whole series: it is handed to the evaluator as one closing series, so - /// the window reports on it once, and its chunk waits in the node's queue at its own estimate. - /// The fragments and the merge's workspace stay charged for the whole flush and go back when it - /// ends; the queued answers stay charged until the caller pulls them. - fn flush_merge(&mut self, index: usize) -> Result<()> { - let path = self.state.paths[index].clone(); - let Some(merge) = self.state.runtimes[index].merge.take() else { - return Ok(()); - }; - if merge.fragments.is_empty() { - // Nothing arrived, so there is nothing to merge and nothing to answer: the window's - // evaluator never read a series and the charge goes back. - return self.settle_merge(index, merge.bytes); - } - // The workspace the merge allocates is charged before it runs, from the rows, segments and - // fields of the fragments as they were collected. - let workspace = merge.workspace_bytes(); - let held = merge.bytes.saturating_add(workspace); - let schema = Arc::clone(&merge.schema); - self.state.runtimes[index].buffer.grow(workspace)?; - // The fragments' charge stands for the merged series too -- a merged run is a slice of a - // fragment's own segment, so no value is copied -- and the workspace stays charged until - // the flush is over. - let merged = merge_same_labelset(&schema, merge.fragments)?; - for series in merged { - let output = { - let runtime = &mut self.state.runtimes[index]; - let Some(evaluator) = runtime.window.as_mut() else { - return Err(closed(&path)); - }; - let metrics = Arc::clone(&runtime.buffer.metrics); - let _timer = metrics.timer(); - evaluator.push_series(&series, true)? - }; - if let Some(output) = output { - let runtime = &mut self.state.runtimes[index]; - runtime - .buffer - .metrics - .record_grid_points(output.series.time.len()); - runtime - .buffer - .metrics - .record_output(1, valid_values(&output)); - runtime.track(&path, &output)?; - let retained = runtime - .window - .as_ref() - .map_or(0, RangeEvaluator::retained_bytes); - let emitted = runtime.emitted_bytes(); - runtime - .buffer - .resize(retained.saturating_add(emitted).saturating_add(held))?; - runtime.enqueue_output(output)?; - } - } - // The fragments and the merge's workspace go back, and whatever the flush produced stays - // charged in the node's queue until the caller pulls it. - self.settle_merge(index, held) - } - - /// Gives back the `give_back` bytes the flush took, leaving the rest of the node's - /// reservation alone. - /// - /// It must not resize to a figure computed here: the chunks the flush queued are still charged - /// to this node, and they stay charged until the pull that hands each of them up returns its - /// estimate. - fn settle_merge(&mut self, index: usize, give_back: usize) -> Result<()> { - self.state.runtimes[index].buffer.shrink(give_back) - } - /// The next chunk of the node at `index`. fn pull(&mut self, index: usize) -> Result> { let tag = self.state.nodes[index].tag; @@ -4084,10 +3662,10 @@ impl NativeExecutor { runtime.instant(path, chunk) } - /// Queues a source chunk at the input node `path` names, or at the tree's only one. - fn push_source(&mut self, path: Option<&NodePath>, chunk: NativeChunk) -> Result<()> { + /// Queues a source chunk at the input leaf `input` names. + fn push_source(&mut self, input: InputId, chunk: NativeChunk) -> Result<()> { self.check_running()?; - let result = self.queue_source(path, chunk); + let result = self.queue_source(input, chunk); if let Err(error) = &result { self.stop(error); } @@ -4095,8 +3673,8 @@ impl NativeExecutor { } /// Queues a source chunk at one input node, checking it against that input's binding. - fn queue_source(&mut self, path: Option<&NodePath>, chunk: NativeChunk) -> Result<()> { - let source = self.source_index(path)?; + fn queue_source(&mut self, input: InputId, chunk: NativeChunk) -> Result<()> { + let source = self.source_index(input)?; let at = &self.state.paths[source]; let Some(binding) = self.state.runtimes[source].source_schema.as_ref() else { return Err(DataFusionError::Execution(format!( @@ -4119,34 +3697,19 @@ impl NativeExecutor { self.state.runtimes[source].enqueue(chunk) } - /// The input node at `path`, or the tree's only input when the caller named none. + /// The node `input` names as an input leaf. /// - /// A tree with a binary node has two inputs, and a chunk queued at "the input" of such a tree - /// cannot be assigned to either of them: the caller's chunks would be matched as the wrong - /// operand, so the ambiguity is refused instead of answered from the first input found. - fn source_index(&self, path: Option<&NodePath>) -> Result { - let Some(path) = path else { - return match self.state.sources.as_slice() { - [only] => Ok(*only), - _ => Err(DataFusionError::Execution(format!( - "the native execution has {} inputs, so a queued chunk has to name the input it \ - belongs to", - self.state.sources.len() - ))), - }; - }; - let index = *self.state.by_path.get(path).ok_or_else(|| { + /// The id counts the leaves left to right, so an id past the last one names no leaf at all: a + /// caller that miscounted its tree is refused rather than answered from whichever leaf the + /// count happened to reach. + fn source_index(&self, input: InputId) -> Result { + let index = *self.state.sources.get(input.index()).ok_or_else(|| { DataFusionError::Execution(format!( - "the native tree has no node at {path} to queue a chunk at" + "the native execution has {} inputs, and no input {} was queued at", + self.state.sources.len(), + input.index() )) })?; - let tag = self.state.nodes[index].tag; - if tag != NodeTag::Input { - return Err(DataFusionError::Execution(format!( - "the node at {path} is a {} node, and only an input node takes queued chunks", - tag.describe() - ))); - } Ok(index) } @@ -4186,8 +3749,7 @@ impl NativeExecutor { /// from "no more input" by that chunk alone. /// /// `admit` is handed the bytes a label change can write before the change builds them; an error -/// from it stops the map. -fn map_chunk( +/// from it stops the map.fn map_chunk( schema: &Arc, chunk: &EvaluatedChunk, ops: &[MapOp], @@ -4277,53 +3839,48 @@ fn evaluated_values(chunk: &NativeChunk) -> usize { } } -/// The error for a window whose evaluator this execution already closed. -fn closed(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!("the window at {path} is already closed")) -} - /// The error for a chunk of another series arriving while one is being selected. /// /// [`select_instants`] reads one whole axis, so the buffered chunks have to be one series: a chunk /// of another series has nowhere to go, and appending it would select its samples as though they /// were the open series'. -fn series_changed(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!( - "the instant selection at {path} combines one series into one axis, and a chunk of another \ - series arrived before the first one ended" - )) +fn series_changed() -> DataFusionError { + DataFusionError::Execution( + "the instant selection combines one series into one axis, and a chunk of another series \ + arrived before the first one ended" + .to_string(), + ) } /// The error for a series an instant selection has no labels to report it under. /// /// A source bound in label mode carries its identity encoded and no decoded labels, and a chunk /// that is handed on has to name the series it holds. -fn no_labels(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!( - "the instant selection at {path} requires materialized series labels" - )) +fn no_labels() -> DataFusionError { + DataFusionError::Execution( + "the instant selection requires materialized series labels".to_string(), + ) } /// The error for a source row the selection named but the series does not hold. /// /// The selection returns indices into the axis it was given, so a row outside it means the two /// disagree rather than that a sample is missing. -fn missing_row(path: &NodePath, index: usize) -> DataFusionError { +fn missing_row(index: usize) -> DataFusionError { DataFusionError::Execution(format!( - "the instant selection at {path} selected the source row {index}, which the series it read \ - does not hold" + "the instant selection selected the source row {index}, which the series it read does not \ + hold" )) } /// The rejection of an input ordering this runtime cannot execute. /// -/// It names the input that declares the ordering and the ordering found, so a reader can tell a -/// tree this runtime cannot run from one it cannot run correctly. -fn unsupported_ordering(ordering: SeriesOrdering, input: &NodePath) -> DataFusionError { +/// It names the ordering found, so a reader can tell a tree this runtime cannot run from one it +/// cannot run correctly. +fn unsupported_ordering(ordering: SeriesOrdering) -> DataFusionError { DataFusionError::NotImplemented(format!( "the native runtime does not execute {ordering:?} input yet: the nodes here read one series \ - at a time, and the input at {input} is {ordering:?}, so the chunks of two of them can \ - arrive interleaved" + at a time, and an input of that ordering hands over two of them interleaved" )) } @@ -4332,16 +3889,15 @@ fn unsupported_ordering(ordering: SeriesOrdering, input: &NodePath) -> DataFusio /// No merge resolves this one: at `position`, field `field` of the label set `labels` would carry /// two values, which is what Prometheus rejects for two series whose label sets became equal. fn occupied_position( - path: &NodePath, labels: &LabelSet, field: usize, run: &Run, previous: &Run, ) -> DataFusionError { DataFusionError::Execution(format!( - "the node at {path} produces two series under the label set {labels:?} that both hold a \ - value of field {field} at position {position}: one occupies {previous:?} and another \ - {run:?}, and no merge resolves two values of one field at one position", + "the node produces two series under the label set {labels:?} that both hold a value of field \ + {field} at position {position}: one occupies {previous:?} and another {run:?}, and no merge \ + resolves two values of one field at one position", position = run.start.max(previous.start), )) } @@ -4350,42 +3906,45 @@ fn occupied_position( /// /// It is what a window's evaluator reports for the input it reads, said by the node that handed the /// chunks over instead: a tree with no window has no evaluator to hold the series, so the input node -/// is the one node left that can tell a series that ended from one that was cut off. -fn truncated_input(path: &NodePath) -> DataFusionError { +/// is the one node left that can tell a series that ended from one that was cut off. The input is +/// named by the routing slot its chunks are handed to, which is the one identity a caller feeding +/// the execution knows it by. +fn truncated_input(input: InputId) -> DataFusionError { DataFusionError::Execution(format!( - "the input at {path} ended before the last chunk of a series: the last chunk it handed over \ - did not say `is_last`" + "the input at routing slot {} ended before the last chunk of a series: the last chunk it \ + handed over did not say `is_last`", + input.index() )) } /// The rejection of a node kind this runtime does not execute yet. /// -/// It names the kind and the node's position: the position is what the metrics and `EXPLAIN` call -/// the same node, so a reader can find it in the tree. -fn not_implemented(kind: &str, path: &NodePath) -> DataFusionError { +/// It names the kind, so a reader can tell a tree this runtime cannot run from one it cannot run +/// correctly. +fn not_implemented(kind: &str) -> DataFusionError { DataFusionError::NotImplemented(format!( - "the native runtime does not execute {kind} nodes yet: {kind} at {path}" + "the native runtime does not execute {kind} nodes yet" )) } /// The error for a node asked to compute without the runtime its kind holds. -fn no_binary(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!("the node at {path} holds no binary operation")) +fn no_binary() -> DataFusionError { + DataFusionError::Execution("the node holds no binary operation".to_string()) } /// The error for an aggregate node whose fold is missing. -fn no_aggregate(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!("the node at {path} holds no aggregate")) +fn no_aggregate() -> DataFusionError { + DataFusionError::Execution("the node holds no aggregate".to_string()) } /// The error for a node asked to run a vector operation without one. -fn no_vector(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!("the node at {path} holds no vector operation")) +fn no_vector() -> DataFusionError { + DataFusionError::Execution("the node holds no vector operation".to_string()) } /// The error for a node asked to broadcast without the broadcast its kind holds. -fn no_broadcast(path: &NodePath) -> DataFusionError { - DataFusionError::Execution(format!("the node at {path} holds no scalar broadcast")) +fn no_broadcast() -> DataFusionError { + DataFusionError::Execution("the node holds no scalar broadcast".to_string()) } /// The rejection of a scalar broadcast whose matching is not the one a constant can take. @@ -4394,24 +3953,24 @@ fn no_broadcast(path: &NodePath) -> DataFusionError { /// the matching modifiers of a vector–scalar operation at all. A tree that carries them therefore /// describes an operation that is not the one it would compute, which is refused rather than run /// as if the modifiers were absent. -fn matched_scalar(path: &NodePath) -> DataFusionError { - DataFusionError::NotImplemented(format!( +fn matched_scalar() -> DataFusionError { + DataFusionError::NotImplemented( "the native runtime does not execute a scalar broadcast whose constant carries matching \ - modifiers: a constant has no labels to match, so the operation this Binary at {path} \ - describes is not the one it would compute" - )) + modifiers: a constant has no labels to match, so the operation this Binary describes is \ + not the one it would compute" + .to_string(), + ) } /// The rejection of a binary node whose operands are neither two evaluated vectors nor one /// evaluated vector and one constant. /// -/// `operands` says which shape arrived and why that shape has no path here, so a tree a later -/// stage compiles is refused for a reason a reader can act on. The kind and the position are named -/// the way every other rejection in this runtime names them. -fn unsupported_binary(path: &NodePath, operands: &str) -> DataFusionError { +/// `operands` says which shape arrived and why that shape has no place here, so a tree a later +/// stage compiles is refused for a reason a reader can act on. +fn unsupported_binary(operands: &str) -> DataFusionError { DataFusionError::NotImplemented(format!( "the native runtime does not execute this Binary node: it reads {operands}, and this \ - runtime reads two evaluated vectors or one evaluated vector and one constant, at {path}" + runtime reads two evaluated vectors or one evaluated vector and one constant" )) } @@ -4429,7 +3988,6 @@ fn unsupported_binary(path: &NodePath, operands: &str) -> DataFusionError { /// match — a constant operand that is not a literal scalar node, and a constant carrying matching /// modifiers — are reachable through a validated tree. fn broadcast_runtime( - path: &NodePath, left: &NativeNode, right: &NativeNode, matching: &BinaryMatching, @@ -4452,15 +4010,13 @@ fn broadcast_runtime( // executor test `an_unimplemented_node_is_reported_not_panicked` builds this pair directly. (NativeType::Scalar(_), NativeType::Scalar(_)) => { return Err(unsupported_binary( - path, "two constants, which broadcast over no vector", )); } // Raw samples or aggregate state: `validate` already refused either as a binary operand, so - // this arm is the exhaustiveness backstop rather than a reachable path. + // this arm is the exhaustiveness backstop rather than a reachable case. (_, _) => { return Err(unsupported_binary( - path, "an operand that is neither evaluated samples nor a constant, which no validated \ tree binds as a binary operand", )); @@ -4470,11 +4026,12 @@ fn broadcast_runtime( // NotImplemented rather than Execution: this is a shape the runtime does not support, not a // tree whose values are wrong. A caller that falls back to another planner on NotImplemented // then still can, which an Execution would deny it. - return Err(DataFusionError::NotImplemented(format!( - "the native runtime does not execute the Binary node at {path}, which broadcasts an \ - operand that is not a literal scalar: the constant side has to be a Scalar node, and a \ - computed constant such as `1 < bool 2` is not one until it is folded" - ))); + return Err(DataFusionError::NotImplemented( + "the native runtime does not execute this Binary node, which broadcasts an operand that \ + is not a literal scalar: the constant side has to be a Scalar node, and a computed \ + constant such as `1 < bool 2` is not one until it is folded" + .to_string(), + )); }; // A constant has no labels, so the modifiers a vector–vector operation would use have no // meaning here: a tree that carries them describes another operation than the one it computes. @@ -4482,11 +4039,11 @@ fn broadcast_runtime( || matching.cardinality != Cardinality::OneToOne || matching.fill != FillValues::default() { - return Err(matched_scalar(path)); + return Err(matched_scalar()); } Ok(BroadcastRuntime { operand: scalar_operand(*value), - op: broadcast_op(path, matching)?, + op: broadcast_op(matching)?, sample_schema: Arc::clone(vector), }) } @@ -4496,13 +4053,13 @@ fn broadcast_runtime( /// A comparison is a filter unless it was bound as `bool`, and a filter keeps the label set it /// read while every other operation drops the metric name. Arithmetic has no `bool` form, so one /// bound that way describes a node the compiler does not produce and is refused. -fn broadcast_op(path: &NodePath, matching: &BinaryMatching) -> Result { +fn broadcast_op(matching: &BinaryMatching) -> Result { let arithmetic = |op: ArithmeticOp| { if matching.return_bool { - Err(DataFusionError::Execution(format!( - "the binary node at {path} carries `bool` on an arithmetic operator, which has no \ - bool form" - ))) + Err(DataFusionError::Execution( + "the binary node carries `bool` on an arithmetic operator, which has no bool form" + .to_string(), + )) } else { Ok(BroadcastOp::Arithmetic(op)) } @@ -4537,19 +4094,12 @@ fn broadcast_op(path: &NodePath, matching: &BinaryMatching) -> Result DataFusionError { - DataFusionError::Execution(format!( - "the operands and the kernel's workspace of the binary node at {path} do not fit one \ - address space: the bytes it would reserve overflow" - )) -} - -/// The error for a node the flattened tree lost its input for. -fn missing_input(tag: NodeTag) -> DataFusionError { - DataFusionError::Execution(format!( - "the native tree has no input for a {} node", - tag.describe() - )) +fn reservation_overflow() -> DataFusionError { + DataFusionError::Execution( + "the operands and the kernel's workspace of the binary node do not fit one address space: \ + the bytes it would reserve overflow" + .to_string(), + ) } fn series_error(error: datatypes::error::Error) -> DataFusionError { diff --git a/src/promql/src/native/execute/tests.rs b/src/promql/src/native/execute/tests.rs index 69ebef36959..1e39155a6cf 100644 --- a/src/promql/src/native/execute/tests.rs +++ b/src/promql/src/native/execute/tests.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use datafusion::error::DataFusionError; use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; -use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue}; +use datafusion::physical_plan::metrics::MetricValue; use datatypes::arrow::array::{Array, Float64Array, UInt32Array}; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; use datatypes::series::histogram::{ @@ -34,9 +34,9 @@ use datatypes::series::time::{EvalGrid, GridRange, GridSpan, GridSpec, TimeAxis} use datatypes::series::{Label, MetricName, SeriesOrdering}; use super::{ - BinaryRuntime, Emitted, NativeChunk, NativeExecutionState, NativeExecutor, NodeBuffer, - NodeRuntime, absorb_rebuilds, evaluated_bytes, grid_end_ms, map_chunk, operand_chunk_bytes, - raw_bytes, vector_workspace_bytes, + BinaryRuntime, Emitted, InputId, NativeChunk, NativeExecutionState, NativeExecutor, NodeBuffer, + NodeRuntime, RuntimeTreeHandle, absorb_rebuilds, evaluated_bytes, grid_end_ms, map_chunk, + operand_chunk_bytes, raw_bytes, vector_workspace_bytes, }; use crate::evaluated::EvaluatedChunk; use crate::grid_series::merge_same_labelset; @@ -687,12 +687,8 @@ fn bounded_pool(bytes: usize) -> Arc { Arc::new(GreedyMemoryPool::new(bytes)) } -fn executor( - tree: &NativeTree, - pool: Arc, - set: &ExecutionPlanMetricsSet, -) -> NativeExecutor { - NativeExecutor::new(NativeExecutionState::try_new(tree, pool, set, 0).unwrap()) +fn executor(tree: &NativeTree, pool: Arc) -> NativeExecutor { + NativeExecutor::new(NativeExecutionState::try_new(tree, pool).unwrap()) } /// Pulls until the tree has nothing settled, then closes it. @@ -791,16 +787,29 @@ fn decided(chunks: &[EvaluatedChunk]) -> Vec<(usize, Option)> { .collect() } -fn count(set: &ExecutionPlanMetricsSet, path: &NodePath, metric: &str) -> usize { - let name = path.metric_name(metric); - match set.clone_inner().sum_by_name(&name) { +/// The count `metric` of the node called `name`, exported fresh from `runtime`. +/// +/// The handles are read from the execution's own runtime tree, so a read here sees what the nodes +/// recorded even after the execution stopped or was dropped. +fn count(runtime: &RuntimeTreeHandle, name: &str, metric: &str) -> usize { + let full = format!("native.{name}.{metric}"); + let exported = runtime + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .export_metrics(0); + match exported.sum_by_name(&full) { Some(MetricValue::Count { count, .. }) => count.value(), - other => panic!("no count named {name}: {other:?}"), + other => panic!("no count named {full}: {other:?}"), } } -fn gauge(set: &ExecutionPlanMetricsSet, name: &str) -> usize { - match set.clone_inner().sum_by_name(name) { +/// The gauge called `name` — its full exported name — read fresh from `runtime`. +fn gauge(runtime: &RuntimeTreeHandle, name: &str) -> usize { + let exported = runtime + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .export_metrics(0); + match exported.sum_by_name(name) { Some(MetricValue::Gauge { gauge, .. }) => gauge.value(), other => panic!("no gauge named {name}: {other:?}"), } @@ -823,8 +832,8 @@ fn a_window_over_raw_input_evaluates_onto_its_own_grid() { true, ); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - execution.push_raw(chunk.clone()).unwrap(); + let mut execution = executor(&tree, pool()); + execution.push_raw(InputId::new(0), chunk.clone()).unwrap(); let out = run(&mut execution); // The same samples, evaluated by the evaluator this runtime wraps. @@ -905,8 +914,10 @@ fn a_window_map_window_chain_keeps_each_layer_on_its_own_grid() { // Down to the map, on its own: the map's chunks stay addressed on the inner window's grid. let map_tree = NativeTree::try_new(mapped()).unwrap(); - let mut map_execution = executor(&map_tree, pool(), &ExecutionPlanMetricsSet::new()); - map_execution.push_raw(chunk.clone()).unwrap(); + let mut map_execution = executor(&map_tree, pool()); + map_execution + .push_raw(InputId::new(0), chunk.clone()) + .unwrap(); let mapped_chunks = run(&mut map_execution); assert_eq!(mapped_chunks.len(), 1); for chunk in &mapped_chunks { @@ -926,8 +937,10 @@ fn a_window_map_window_chain_keeps_each_layer_on_its_own_grid() { // The whole chain: only the outer window's grid leaves the tree. let chain_tree = NativeTree::try_new(chain()).unwrap(); - let mut chain_execution = executor(&chain_tree, pool(), &ExecutionPlanMetricsSet::new()); - chain_execution.push_raw(chunk.clone()).unwrap(); + let mut chain_execution = executor(&chain_tree, pool()); + chain_execution + .push_raw(InputId::new(0), chunk.clone()) + .unwrap(); let out = run(&mut chain_execution); assert_eq!(out.len(), 1); for chunk in &out { @@ -1022,9 +1035,9 @@ fn a_downstream_window_emits_before_its_input_series_ends() { let head = evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0], false); let tail = evaluated_chunk(&schema, 3, vec![4.0, 5.0], true); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); - execution.push_evaluated(head).unwrap(); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); + execution.push_evaluated(InputId::new(0), head).unwrap(); let mut early = Vec::new(); while let Some(chunk) = execution.next_chunk().unwrap() { @@ -1049,13 +1062,13 @@ fn a_downstream_window_emits_before_its_input_series_ends() { } // The open outer series is still accounted for: the pieces handed over are what it holds. assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), + count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 0, "a series closed before its last piece arrived" ); // The closing piece: the rest of the outer grid, and the one chunk that closes the series. - execution.push_evaluated(tail).unwrap(); + execution.push_evaluated(InputId::new(0), tail).unwrap(); let mut out = early; while let Some(chunk) = execution.next_chunk().unwrap() { out.push(chunk); @@ -1068,7 +1081,7 @@ fn a_downstream_window_emits_before_its_input_series_ends() { assert_eq!(out.iter().filter(|chunk| chunk.is_last).count(), 1); assert!(out.last().unwrap().is_last); assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), + count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 1, "the outer series closed once per piece instead of once in total" ); @@ -1082,14 +1095,12 @@ fn a_downstream_window_emits_before_its_input_series_ends() { Arc::clone(&outer_grid), )) .unwrap(); - let mut whole = executor(&whole_tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut whole = executor(&whole_tree, pool()); whole - .push_evaluated(evaluated_chunk( - &schema, - 0, - vec![1.0, 2.0, 3.0, 4.0, 5.0], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0, 5.0], true), + ) .unwrap(); let whole = run(&mut whole); assert_eq!(decided(&out), decided(&whole)); @@ -1149,12 +1160,7 @@ fn an_unimplemented_node_is_reported_not_panicked() { for (kind, path, reason, root) in cases { // Legality stays `validate`'s: the plan accepts these trees, the runtime does not run them. let tree = NativeTree::try_new(root).expect("the bound tree is legal"); - let error = match NativeExecutionState::try_new( - &tree, - pool(), - &ExecutionPlanMetricsSet::new(), - 0, - ) { + let error = match NativeExecutionState::try_new(&tree, pool()) { Ok(_) => panic!("the runtime accepted a {kind} node"), Err(error) => error, }; @@ -1194,17 +1200,26 @@ fn each_node_records_its_own_metrics() { outer_grid, ); let tree = NativeTree::try_new(root).unwrap(); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); let schema = raw_schema(); execution - .push_raw(raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false), + ) .unwrap(); execution - .push_raw(raw_chunk(&schema, &[120_000], &[3.0], false)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[120_000], &[3.0], false), + ) .unwrap(); execution - .push_raw(raw_chunk(&schema, &[180_000], &[4.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[180_000], &[4.0], true), + ) .unwrap(); let out = run(&mut execution); // The outer window reports the two points of its grid once its input ended, in the one chunk of @@ -1212,30 +1227,44 @@ fn each_node_records_its_own_metrics() { assert_eq!(addressed(&out), vec![(0, vec![Some(1.0), Some(3.0)])]); assert!(out.last().unwrap().is_last); - let outer = NodePath::root(); - let inner = outer.input(); - let source = inner.input(); - // Three source chunks reach the inner window, which hands one chunk on per chunk it took in. // The outer window takes those three pieces in and reports the one series they merge into. - assert_eq!(count(&set, &source, NodeMetric::INPUT_BATCHES), 3); - assert_eq!(count(&set, &source, NodeMetric::OUTPUT_BATCHES), 3); - assert_eq!(count(&set, &inner, NodeMetric::INPUT_BATCHES), 3); - assert_eq!(count(&set, &outer, NodeMetric::INPUT_BATCHES), 3); - assert_eq!(count(&set, &inner, NodeMetric::OUTPUT_BATCHES), 3); - assert_eq!(count(&set, &outer, NodeMetric::OUTPUT_BATCHES), 1); + assert_eq!( + count(&metrics, "root.input.input", NodeMetric::INPUT_BATCHES), + 3 + ); + assert_eq!( + count(&metrics, "root.input.input", NodeMetric::OUTPUT_BATCHES), + 3 + ); + assert_eq!(count(&metrics, "root.input", NodeMetric::INPUT_BATCHES), 3); + assert_eq!(count(&metrics, "root", NodeMetric::INPUT_BATCHES), 3); + assert_eq!(count(&metrics, "root.input", NodeMetric::OUTPUT_BATCHES), 3); + assert_eq!(count(&metrics, "root", NodeMetric::OUTPUT_BATCHES), 1); // The grid points each window decided, which is its own output range: four on the inner grid // and two on the outer one, which is the figure that tells the two nodes apart. - assert_eq!(count(&set, &inner, NodeMetric::EVALUATED_GRID_POINTS), 4); - assert_eq!(count(&set, &outer, NodeMetric::EVALUATED_GRID_POINTS), 2); + assert_eq!( + count(&metrics, "root.input", NodeMetric::EVALUATED_GRID_POINTS), + 4 + ); + assert_eq!( + count(&metrics, "root", NodeMetric::EVALUATED_GRID_POINTS), + 2 + ); // The one series closes once, on the chunk carrying `is_last`. - assert_eq!(count(&set, &source, NodeMetric::COMPLETED_SERIES), 1); - assert_eq!(count(&set, &inner, NodeMetric::COMPLETED_SERIES), 1); - assert_eq!(count(&set, &outer, NodeMetric::COMPLETED_SERIES), 1); + assert_eq!( + count(&metrics, "root.input.input", NodeMetric::COMPLETED_SERIES), + 1 + ); + assert_eq!( + count(&metrics, "root.input", NodeMetric::COMPLETED_SERIES), + 1 + ); + assert_eq!(count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 1); // Per node, not one total for the tree. assert_ne!( - count(&set, &inner, NodeMetric::EVALUATED_GRID_POINTS), - count(&set, &outer, NodeMetric::EVALUATED_GRID_POINTS) + count(&metrics, "root.input", NodeMetric::EVALUATED_GRID_POINTS), + count(&metrics, "root", NodeMetric::EVALUATED_GRID_POINTS) ); } @@ -1244,10 +1273,10 @@ fn each_node_records_its_own_metrics() { fn an_error_releases_the_reservation() { let tree = NativeTree::try_new(window(input(), grid(0, 60_000, 4))).unwrap(); let pool = pool(); - let set = ExecutionPlanMetricsSet::new(); let schema = raw_schema(); - let mut execution = executor(&tree, Arc::clone(&pool), &set); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = execution.state().runtime_tree(); assert_eq!(pool.reserved(), 0); assert_eq!( execution.state().nodes_holding_windows(), @@ -1255,7 +1284,10 @@ fn an_error_releases_the_reservation() { "a window that has not been released is what the drop below has to give back" ); execution - .push_raw(raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false), + ) .unwrap(); assert!( pool.reserved() > 0, @@ -1274,7 +1306,10 @@ fn an_error_releases_the_reservation() { // A second chunk of the same series whose timestamps do not increase: the evaluator rejects it // after the node already held input for it. execution - .push_raw(raw_chunk(&schema, &[0, 60_000], &[3.0, 4.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 60_000], &[3.0, 4.0], true), + ) .unwrap(); let error = execution.next_chunk().unwrap_err(); assert!(error.to_string().contains("not increasing"), "{error}"); @@ -1290,10 +1325,10 @@ fn an_error_releases_the_reservation() { 0, "the stopped execution still holds a window's samples while its reservation reads zero" ); - let current = NodePath::root().metric_name(NodeMetric::CURRENT_RESERVED_BYTES); - assert_eq!(gauge(&set, ¤t), 0); - let peak = NodePath::root().partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 0); - assert!(gauge(&set, &peak) >= held); + let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES); + assert_eq!(gauge(&metrics, ¤t), 0); + let peak = format!("native.p0.root.{}", NodeMetric::PEAK_RESERVED_BYTES); + assert!(gauge(&metrics, &peak) >= held); // A stopped execution is not resumed. let again = execution.next_chunk().unwrap_err(); assert!( @@ -1302,9 +1337,9 @@ fn an_error_releases_the_reservation() { ); // Dropping an execution that never finished returns its bytes too. - let mut unfinished = executor(&tree, Arc::clone(&pool), &set); + let mut unfinished = executor(&tree, Arc::clone(&pool)); unfinished - .push_raw(raw_chunk(&schema, &[0], &[1.0], false)) + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) .unwrap(); let _ = unfinished.next_chunk().unwrap(); assert!( @@ -1324,13 +1359,13 @@ fn an_error_releases_the_reservation() { fn a_pool_that_cannot_admit_a_chunk_releases_what_it_held() { let tree = NativeTree::try_new(window(input(), grid(0, 60_000, 4))).unwrap(); let schema = raw_schema(); - let set = ExecutionPlanMetricsSet::new(); // Room for the first chunk and what the window keeps of it, not for the second one. let pool = bounded_pool(16 << 10); - let mut execution = executor(&tree, Arc::clone(&pool), &set); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = execution.state().runtime_tree(); execution - .push_raw(raw_chunk(&schema, &[0], &[1.0], false)) + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) .unwrap(); drop(execution.next_chunk().unwrap()); let held = pool.reserved(); @@ -1340,7 +1375,10 @@ fn a_pool_that_cannot_admit_a_chunk_releases_what_it_held() { let timestamps: Vec = (0..100_000).collect(); let values = vec![1.0; 100_000]; let error = execution - .push_raw(raw_chunk(&schema, ×tamps, &values, true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, ×tamps, &values, true), + ) .expect_err("a chunk larger than the pool was admitted"); assert!( matches!(error, DataFusionError::ResourcesExhausted(_)), @@ -1351,10 +1389,10 @@ fn a_pool_that_cannot_admit_a_chunk_releases_what_it_held() { // while the executor is still alive. assert_eq!(execution.state().reserved_bytes(), 0); assert_eq!(pool.reserved(), 0); - let current = NodePath::root().metric_name(NodeMetric::CURRENT_RESERVED_BYTES); - assert_eq!(gauge(&set, ¤t), 0); - let peak = NodePath::root().partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 0); - assert!(gauge(&set, &peak) >= held); + let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES); + assert_eq!(gauge(&metrics, ¤t), 0); + let peak = format!("native.p0.root.{}", NodeMetric::PEAK_RESERVED_BYTES); + assert!(gauge(&metrics, &peak) >= held); // A stopped execution is not resumed. let again = execution.next_chunk().unwrap_err(); assert!( @@ -1408,15 +1446,15 @@ fn a_collector_refuses_the_malformed_streams_a_streaming_window_refuses() { ]; for (what, first, second, phrase) in cases { let pool = pool(); - let mut execution = executor(&tree(), Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), Arc::clone(&pool)); // The chunk that only opens the series is what a stream hands over first, and it is legal: // the collector keeps it and reports nothing. - execution.push_evaluated(first).unwrap(); + execution.push_evaluated(InputId::new(0), first).unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "{what}: the collector reported a piece before its input ended" ); - execution.push_evaluated(second).unwrap(); + execution.push_evaluated(InputId::new(0), second).unwrap(); let error = match execution.next_chunk() { Ok(None) => panic!("{what} was collected as if it were one series"), Ok(Some(chunk)) => panic!("{what} reached the evaluator as {:?}", decided(&[chunk])), @@ -1486,9 +1524,9 @@ fn a_collector_accepts_forward_runs_and_a_close_without_a_row() { ), ]; for (what, chunks, expected) in cases { - let mut execution = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), pool()); for chunk in chunks { - execution.push_evaluated(chunk).unwrap(); + execution.push_evaluated(InputId::new(0), chunk).unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "{what}: the collector reported a piece before its input ended" @@ -1532,18 +1570,15 @@ fn an_input_that_ends_inside_a_collected_series_is_reported() { .unwrap(); let schema = evaluated_schema(Arc::clone(&inner_grid)); let pool = pool(); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, Arc::clone(&pool), &set); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = execution.state().runtime_tree(); // One piece of a series that never says it ended: the collector keeps it and reports nothing. execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(-1.0)], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric("a", "x"), 0, vec![Some(-1.0)], false), + ) .unwrap(); assert!( execution @@ -1576,8 +1611,8 @@ fn an_input_that_ends_inside_a_collected_series_is_reported() { "the stopped execution still holds a window's samples" ); // The window's own gauge, which is the node that collected the fragment, reads zero. - let current = NodePath::root().metric_name(NodeMetric::CURRENT_RESERVED_BYTES); - assert_eq!(gauge(&set, ¤t), 0); + let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES); + assert_eq!(gauge(&metrics, ¤t), 0); // A stopped execution is not resumed. let again = execution.next_chunk().unwrap_err(); assert!( @@ -1642,26 +1677,32 @@ fn two_series_onto_one_label_set_after_a_map_are_merged_into_one_series() { (RangeFunction::CountOverTime, 2.0), ] { for (shape, tree) in shapes(function) { - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // Each source series is closed properly, and each holds its value where the other holds // none, so the pair is one logical series and one answer. execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(1.0), None], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 0, + vec![Some(1.0), None], + true, + ), + ) .unwrap(); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("b", "x"), - 0, - vec![None, Some(3.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("b", "x"), + 0, + vec![None, Some(3.0)], + true, + ), + ) .unwrap(); // Nothing settles while the pieces arrive: the merged series is what the window evaluates, // and it does not exist before the input ends. @@ -1706,15 +1747,18 @@ fn two_series_onto_one_label_set_after_a_map_are_rejected() { }; let schema = evaluated_schema(Arc::clone(&inner_grid)); - let mut execution = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(1.0), None], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 0, + vec![Some(1.0), None], + true, + ), + ) .unwrap(); // The first series closed, and the window above collects it rather than reporting it. assert!( @@ -1722,13 +1766,16 @@ fn two_series_onto_one_label_set_after_a_map_are_rejected() { "the window above the map reported a piece before its input ended" ); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("b", "x"), - 0, - vec![Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("b", "x"), + 0, + vec![Some(3.0), Some(4.0)], + true, + ), + ) .unwrap(); let error = match execution.next_chunk() { Ok(None) => panic!("the second series was accepted at a position the first one holds"), @@ -1786,31 +1833,37 @@ fn an_empty_tag_and_an_absent_one_are_one_label_set_to_the_window_above() { let schema = raw_schema(); let push_both = |execution: &mut NativeExecutor| { execution - .push_raw(raw_chunk_of( - &schema, - b"series-a", - host_metric(Some("")), - &[0], - &[1.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-a", + host_metric(Some("")), + &[0], + &[1.0], + true, + ), + ) .unwrap(); execution - .push_raw(raw_chunk_of( - &schema, - b"series-b", - host_metric(None), - &[60_000], - &[3.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-b", + host_metric(None), + &[60_000], + &[3.0], + true, + ), + ) .unwrap(); }; // The inner window alone hands both series on, and they are two series: the empty tag survives // `last_over_time`, which keeps the name and every tag. let inner_tree = NativeTree::try_new(inner()).unwrap(); - let mut inner_execution = executor(&inner_tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut inner_execution = executor(&inner_tree, pool()); push_both(&mut inner_execution); let pieces = run(&mut inner_execution); assert_eq!( @@ -1825,7 +1878,7 @@ fn an_empty_tag_and_an_absent_one_are_one_label_set_to_the_window_above() { // The chain merges them before the outer window reads them. let tree = NativeTree::try_new(chain()).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); push_both(&mut execution); let out = run(&mut execution); assert_eq!( @@ -1858,18 +1911,15 @@ fn a_window_that_merges_its_input_releases_what_it_collected() { )) .unwrap(); let schema = evaluated_schema(Arc::clone(&inner_grid)); - let set = ExecutionPlanMetricsSet::new(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &set); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = execution.state().runtime_tree(); for (name, values) in [("a", vec![Some(1.0), None]), ("b", vec![None, Some(3.0)])] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric(name, "x"), - 0, - values, - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), + ) .unwrap(); } // Pulling takes both fragments out of the queue and into the window, which charges them. @@ -1892,11 +1942,11 @@ fn a_window_that_merges_its_input_releases_what_it_collected() { assert_eq!(execution.state().reserved_bytes(), 0); assert_eq!(pool.reserved(), 0); assert_eq!(execution.state().nodes_holding_windows(), 0); - let current = NodePath::root().metric_name(NodeMetric::CURRENT_RESERVED_BYTES); - assert_eq!(gauge(&set, ¤t), 0); - let peak = NodePath::root().partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 0); + let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES); + assert_eq!(gauge(&metrics, ¤t), 0); + let peak = format!("native.p0.root.{}", NodeMetric::PEAK_RESERVED_BYTES); assert!( - gauge(&set, &peak) >= window_held, + gauge(&metrics, &peak) >= window_held, "the peak does not cover the input the window collected" ); } @@ -1931,9 +1981,9 @@ fn a_pool_that_cannot_admit_a_collected_fragment_releases_what_it_held() { // Room for the chunk the source queue holds, which is what the pool admits first, and nothing // for the label sets the map records or the fragment the window collects. let pool = bounded_pool(evaluated_bytes(&chunk) + 1); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated(chunk) + .push_evaluated(InputId::new(0), chunk) .expect("the source queue holds the chunk"); let error = execution @@ -1980,19 +2030,16 @@ fn a_pool_that_cannot_admit_the_merge_workspace_releases_what_it_held() { let push_both = |execution: &mut NativeExecutor| { for (name, values) in [("a", vec![Some(-1.0), None]), ("b", vec![None, Some(-3.0)])] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric(name, "x"), - 0, - values, - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), + ) .unwrap(); } }; // What the chain holds once its input is collected, measured where the pool admits it. - let mut reference = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut reference = executor(&tree(), pool()); push_both(&mut reference); assert!( reference.next_chunk().unwrap().is_none(), @@ -2004,7 +2051,7 @@ fn a_pool_that_cannot_admit_the_merge_workspace_releases_what_it_held() { // Room for exactly what the collected input holds, and not for the merge's workspace. let pool = bounded_pool(measured + 64); - let mut execution = executor(&tree(), Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), Arc::clone(&pool)); push_both(&mut execution); assert!( execution.next_chunk().unwrap().is_none(), @@ -2069,16 +2116,19 @@ fn two_series_onto_one_label_set_after_a_window_are_rejected() { let schema = raw_schema(); let counter = [0, 60_000, 120_000, 180_000, 240_000]; - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw(raw_chunk_of( - &schema, - b"series-a", - host_metric(Some("")), - &counter, - &[1.0, 2.0, 3.0, 4.0, 5.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-a", + host_metric(Some("")), + &counter, + &[1.0, 2.0, 3.0, 4.0, 5.0], + true, + ), + ) .unwrap(); let first = execution .next_chunk() @@ -2099,14 +2149,17 @@ fn two_series_onto_one_label_set_after_a_window_are_rejected() { // The second series is a different one to the scan and the same one to `rate`. execution - .push_raw(raw_chunk_of( - &schema, - b"series-b", - host_metric(None), - &counter, - &[6.0, 7.0, 8.0, 9.0, 10.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-b", + host_metric(None), + &counter, + &[6.0, 7.0, 8.0, 9.0, 10.0], + true, + ), + ) .unwrap(); let error = match execution.next_chunk() { Ok(None) => panic!("the second series settled nothing at all"), @@ -2126,34 +2179,39 @@ fn two_series_onto_one_label_set_after_a_window_are_rejected() { // Two series carrying different hosts stay distinct, which is what a rate over a label that // survives the rewrite looks like. - let mut distinct = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut distinct = executor(&tree, pool()); distinct - .push_raw(raw_chunk_of( - &schema, - b"series-a", - host_metric(Some("")), - &counter, - &[1.0, 2.0, 3.0, 4.0, 5.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-a", + host_metric(Some("")), + &counter, + &[1.0, 2.0, 3.0, 4.0, 5.0], + true, + ), + ) .unwrap(); distinct - .push_raw(raw_chunk_of( - &schema, - b"series-b", - MetricName::try_new( - Some("requests".into()), - vec![Label { - name: "host".into(), - value: "web-2".into(), - }], - ) - .unwrap(), - &counter, - &[6.0, 7.0, 8.0, 9.0, 10.0], - true, - )) - .unwrap(); + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + b"series-b", + MetricName::try_new( + Some("requests".into()), + vec![Label { + name: "host".into(), + value: "web-2".into(), + }], + ) + .unwrap(), + &counter, + &[6.0, 7.0, 8.0, 9.0, 10.0], + true, + ), + ) .unwrap(); let out = run(&mut distinct); assert_eq!(out.len(), 2, "two series over two hosts are two series"); } @@ -2173,19 +2231,16 @@ fn two_series_holding_a_value_at_one_position_are_rejected() { .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (name, values) in [ ("a", vec![Some(-1.0), Some(-2.0)]), ("b", vec![Some(-3.0), Some(-4.0)]), ] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric(name, "x"), - 0, - values, - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, false), + ) .unwrap(); } // The first piece is fine on its own: nothing has occupied any position yet. @@ -2227,18 +2282,15 @@ fn a_merge_across_two_closed_series_is_the_root_piece_stream() { .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // Each source series is closed properly: the pieces hold disjoint positions of field 0 and both // end their series. for (name, values) in [("a", vec![Some(-1.0), None]), ("b", vec![None, Some(-3.0)])] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric(name, "x"), - 0, - values, - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), + ) .unwrap(); } // Both pieces come out of the root, one per source series, and neither is refused. @@ -2323,15 +2375,18 @@ fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides .expect("a map-only tree is legal over a contiguous input"); let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(-1.0), None], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 0, + vec![Some(-1.0), None], + false, + ), + ) .unwrap(); let first = execution .next_chunk() @@ -2343,13 +2398,16 @@ fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides // Positions 2 and 3 of field 0, holding a value at 3 only: no position of the field is // occupied twice, so nothing the check looks at is claimed by both series. execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("b", "x"), - 2, - vec![None, Some(-3.0)], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("b", "x"), + 2, + vec![None, Some(-3.0)], + false, + ), + ) .unwrap(); let second = execution .next_chunk() @@ -2397,25 +2455,19 @@ fn a_nan_occupies_its_position_where_a_missing_sample_does_not() { Ok(()) }; for (first, second, accepted) in cases { - let mut execution = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - first.clone(), - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric("a", "x"), 0, first.clone(), false), + ) .unwrap(); drain(&mut execution).expect("the first open chunk is handed on"); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("b", "x"), - 0, - second.clone(), - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric("b", "x"), 0, second.clone(), false), + ) .unwrap(); let outcome = drain(&mut execution); assert_eq!( @@ -2449,15 +2501,18 @@ fn a_series_that_continues_where_it_ended_is_not_a_second_series() { .unwrap(); let schema = evaluated_schema(Arc::clone(&inner_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 0, - vec![Some(-1.0), None], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 0, + vec![Some(-1.0), None], + false, + ), + ) .unwrap(); // Nothing comes out of the collecting window before its input ended, and the piece here does // not end it. @@ -2468,13 +2523,16 @@ fn a_series_that_continues_where_it_ended_is_not_a_second_series() { ); } execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("a", "x"), - 2, - vec![None, Some(-3.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("a", "x"), + 2, + vec![None, Some(-3.0)], + true, + ), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -2510,16 +2568,19 @@ fn a_window_reserves_what_it_retains_not_what_it_consumed() { let peak = |count: usize| { let pool = bounded_pool(32 << 10); - let mut execution = executor(&tree(), Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), Arc::clone(&pool)); let mut peak = 0; for index in 0..count { execution - .push_raw(raw_chunk( - &schema, - &[index as i64], - &[index as f64], - index + 1 == count, - )) + .push_raw( + InputId::new(0), + raw_chunk( + &schema, + &[index as i64], + &[index as f64], + index + 1 == count, + ), + ) .unwrap_or_else(|error| { panic!("chunk {index} of the {count}-chunk series did not fit: {error}") }); @@ -2562,16 +2623,19 @@ fn a_window_retains_the_samples_of_every_field() { let peak = |width: usize| -> usize { let tree = NativeTree::try_new(window_over_fields(width, grid(0, 1, 1024))).unwrap(); let schema = raw_schema_of(field_count(width)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); let (count, mut peak) = (8, 0); for index in 0..count { execution - .push_raw(raw_chunk_repeated( - &schema, - &[index as i64], - &[index as f64], - index + 1 == count, - )) + .push_raw( + InputId::new(0), + raw_chunk_repeated( + &schema, + &[index as i64], + &[index as f64], + index + 1 == count, + ), + ) .unwrap(); drop(execution.next_chunk().unwrap()); peak = peak.max(execution.state().reserved_bytes()); @@ -2609,10 +2673,13 @@ fn a_chunk_with_no_output_keeps_charging_the_emitted_table() { .unwrap(); let schema = raw_schema(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_raw(raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 60_000], &[1.0, 2.0], false), + ) .unwrap(); let first = execution .next_chunk() @@ -2629,7 +2696,10 @@ fn a_chunk_with_no_output_keeps_charging_the_emitted_table() { // Both points are decided, so this chunk decides nothing and the window yields no chunk. execution - .push_raw(raw_chunk(&schema, &[120_000], &[3.0], false)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[120_000], &[3.0], false), + ) .unwrap(); assert!( execution.next_chunk().unwrap().is_none(), @@ -2684,11 +2754,10 @@ fn a_window_over_an_interleaved_input_is_rejected() { Arc::clone(&outer_grid), )) .expect("the binding accepts a window over a map"); - let error = - match NativeExecutionState::try_new(&tree, pool(), &ExecutionPlanMetricsSet::new(), 0) { - Ok(_) => panic!("the runtime accepted an interleaved input under a window"), - Err(error) => error, - }; + let error = match NativeExecutionState::try_new(&tree, pool()) { + Ok(_) => panic!("the runtime accepted an interleaved input under a window"), + Err(error) => error, + }; assert!( matches!(error, DataFusionError::NotImplemented(_)), "{error}" @@ -2714,11 +2783,10 @@ fn an_interleaved_input_is_refused_without_a_window() { vec![MathOp::Abs], )) .expect("the binding accepts a map over an interleaved input"); - let error = - match NativeExecutionState::try_new(&tree, pool(), &ExecutionPlanMetricsSet::new(), 0) { - Ok(_) => panic!("the runtime accepted an interleaved input with no window above it"), - Err(error) => error, - }; + let error = match NativeExecutionState::try_new(&tree, pool()) { + Ok(_) => panic!("the runtime accepted an interleaved input with no window above it"), + Err(error) => error, + }; assert!( matches!(error, DataFusionError::NotImplemented(_)), "{error}" @@ -2810,19 +2878,16 @@ fn a_series_with_no_result_after_a_closed_one_is_passed_over() { )) .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // `abs` drops the name, so both series arrive under the same label set. The first holds a // value; the second holds none at all and has to be passed over rather than refused. for (name, values) in [("a", vec![Some(-7.0)]), ("b", vec![None])] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric(name, "x"), - 0, - values, - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric(name, "x"), 0, values, true), + ) .unwrap(); } @@ -2859,7 +2924,7 @@ fn an_empty_closing_chunk_of_another_series_does_not_close_a_label_set() { )) .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // `a` opens the label set and has not closed it when `b` closes an empty series onto it, and the // chunk that closes `a` itself comes last. @@ -2871,7 +2936,7 @@ fn an_empty_closing_chunk_of_another_series_does_not_close_a_label_set() { let steps = chunks.len(); let mut handed = Vec::new(); for (step, chunk) in chunks.into_iter().enumerate() { - execution.push_evaluated(chunk).unwrap(); + execution.push_evaluated(InputId::new(0), chunk).unwrap(); while let Some(chunk) = execution .next_chunk() .unwrap_or_else(|error| panic!("step {step} was refused: {error}")) @@ -2924,20 +2989,22 @@ fn two_series_with_no_result_do_not_collide() { )) .unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // One sample each: rate over a 5m window with a single sample has no result. for name in ["a", "b"] { execution - .push_raw(raw_chunk_of( - &schema, - name.as_bytes(), - MetricName::try_new(Some(name.into()), vec![]).unwrap(), - &[0], - &[1.0], - true, - )) - .unwrap(); + .push_raw( + InputId::new(0), + raw_chunk_of( + &schema, + name.as_bytes(), + MetricName::try_new(Some(name.into()), vec![]).unwrap(), + &[0], + &[1.0], + true, + ), + ) .unwrap(); } let mut handed = Vec::new(); loop { @@ -3017,8 +3084,8 @@ fn a_sparse_chunk_is_read_at_the_positions_it_holds_through_window_map_window() true, ); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - execution.push_evaluated(chunk).unwrap(); + let mut execution = executor(&tree, pool()); + execution.push_evaluated(InputId::new(0), chunk).unwrap(); let out = run(&mut execution); // The outer window's points: every one of them has a sample in its range, so every one holds @@ -3046,17 +3113,20 @@ fn a_sparse_chunk_occupies_only_the_positions_it_holds() { .unwrap(); let schema = evaluated_schema(Arc::clone(&map_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // Series a holds positions 0, 1 and 5, 6: two runs with a hole in the middle, not one run // covering the hole. execution - .push_evaluated(sparse_chunk( - &schema, - named_metric("a", "x"), - &[(0, 2), (5, 2)], - vec![-1.0, -2.0, -3.0, -4.0], - false, - )) + .push_evaluated( + InputId::new(0), + sparse_chunk( + &schema, + named_metric("a", "x"), + &[(0, 2), (5, 2)], + vec![-1.0, -2.0, -3.0, -4.0], + false, + ), + ) .unwrap(); execution .next_chunk() @@ -3065,13 +3135,10 @@ fn a_sparse_chunk_occupies_only_the_positions_it_holds() { // Position 2 sits in the hole, so series b occupies it and collides with nothing a holds. execution - .push_evaluated(sparse_chunk( - &schema, - named_metric("b", "x"), - &[(2, 1)], - vec![-5.0], - true, - )) + .push_evaluated( + InputId::new(0), + sparse_chunk(&schema, named_metric("b", "x"), &[(2, 1)], vec![-5.0], true), + ) .unwrap(); execution .next_chunk() @@ -3102,11 +3169,14 @@ fn an_emptying_operation_hands_on_nothing_until_it_closes_the_series() { .unwrap(); let schema = evaluated_schema(Arc::clone(&inner_grid)); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // An open chunk that maps to nothing yields nothing at all: it is not handed to the window, // which would refuse an empty delivery inside a series. execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![5.0, 6.0], false)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![5.0, 6.0], false), + ) .unwrap(); assert!( execution.next_chunk().unwrap().is_none(), @@ -3115,7 +3185,10 @@ fn an_emptying_operation_hands_on_nothing_until_it_closes_the_series() { // The closing chunk still closes: one empty chunk reaches the window and closes the series. execution - .push_evaluated(evaluated_chunk(&schema, 2, vec![7.0, 8.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 2, vec![7.0, 8.0], true), + ) .unwrap(); let mut out = Vec::new(); while let Some(chunk) = execution.next_chunk().unwrap() { @@ -3162,10 +3235,13 @@ fn an_input_that_ends_mid_series_is_reported_without_a_window() { ]; for (what, root) in shapes { let tree = NativeTree::try_new(root).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // A chunk that does not close its series: the chunk carrying `is_last` never arrives. execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![-1.0, -2.0], false)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![-1.0, -2.0], false), + ) .unwrap(); let handed = execution .next_chunk() @@ -3206,11 +3282,14 @@ fn a_window_over_histograms_reserves_what_it_retains() { .unwrap(); let schema = raw_schema(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); for (timestamp, is_last) in [(0, false), (10_000, true)] { execution - .push_raw(raw_histogram_chunk(&schema, &[timestamp], 2, is_last)) + .push_raw( + InputId::new(0), + raw_histogram_chunk(&schema, &[timestamp], 2, is_last), + ) .unwrap(); while execution.next_chunk().unwrap().is_some() {} assert_eq!( @@ -3537,15 +3616,15 @@ fn a_bare_selector_selects_instants_onto_its_own_grid() { raw_chunk(&schema, &[120_000, 200_000], &[4.0, 5.0], true), ]; - let mut once = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - once.push_raw(whole).unwrap(); + let mut once = executor(&tree, pool()); + once.push_raw(InputId::new(0), whole).unwrap(); let out = run(&mut once); - let mut piecewise = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut piecewise = executor(&tree, pool()); let mut in_pieces = Vec::new(); for chunk in split { let closing = chunk.is_last; - piecewise.push_raw(chunk).unwrap(); + piecewise.push_raw(InputId::new(0), chunk).unwrap(); // Until the series ends nothing is decided for it, so only the closing chunk reports. let pulled = piecewise.next_chunk().unwrap(); assert_eq!( @@ -3597,9 +3676,12 @@ fn a_sample_at_the_lookback_edge_is_not_selected() { let schema = raw_schema(); let selected = |timestamp: i64| { let tree = NativeTree::try_new(instant(input(), 200, 0, Arc::clone(&out_grid))).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw(raw_chunk(&schema, &[timestamp], &[1.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[timestamp], &[1.0], true), + ) .unwrap(); run(&mut execution) }; @@ -3626,14 +3708,12 @@ fn an_offset_reads_a_different_sample_at_each_grid_point() { let values = |offset_ms: i64| { let tree = NativeTree::try_new(instant(input(), 600_000, offset_ms, Arc::clone(&out_grid))) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw(raw_chunk( - &schema, - &[-60_000, 0, 60_000], - &[1.0, 2.0, 3.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[-60_000, 0, 60_000], &[1.0, 2.0, 3.0], true), + ) .unwrap(); decided(&run(&mut execution)) }; @@ -3664,11 +3744,14 @@ fn a_series_with_no_selectable_sample_closes_once() { let stale = f64::from_bits(common_query::prometheus::PROMETHEUS_STALE_NAN_BITS); let out_grid = grid(0, 60_000, 3); let tree = NativeTree::try_new(instant(input(), 300_000, 0, Arc::clone(&out_grid))).unwrap(); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); let schema = raw_schema(); execution - .push_raw(raw_chunk(&schema, &[0, 60_000], &[stale, stale], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 60_000], &[stale, stale], true), + ) .unwrap(); let out = run(&mut execution); @@ -3694,7 +3777,7 @@ fn a_series_with_no_selectable_sample_closes_once() { "the series lost its identity" ); assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), + count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 1, "the series did not close exactly once" ); @@ -3714,8 +3797,8 @@ fn a_selected_row_keeps_the_fields_that_miss_it_null() { vec![vec![Some(1.0), None], vec![None, Some(4.0)]], true, ); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - execution.push_raw(chunk).unwrap(); + let mut execution = executor(&tree, pool()); + execution.push_raw(InputId::new(0), chunk).unwrap(); let out = run(&mut execution); assert_eq!(out.len(), 1); @@ -3748,9 +3831,9 @@ fn a_position_no_field_fills_is_not_a_row() { let out_grid = grid(0, 60_000, 5); let tree = NativeTree::try_new(instant(input(), 1_000, 0, Arc::clone(&out_grid))).unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw(raw_chunk(&schema, &[0], &[1.0], false)) + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) .unwrap(); // The hole between the two samples decides nothing, and it does not end the series. assert!( @@ -3762,7 +3845,10 @@ fn a_position_no_field_fills_is_not_a_row() { "the open series is not reserved for" ); execution - .push_raw(raw_chunk(&schema, &[240_000], &[5.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[240_000], &[5.0], true), + ) .unwrap(); let out = run(&mut execution); @@ -3802,14 +3888,12 @@ fn a_selected_histogram_keeps_its_start_timestamp() { let out_grid = grid(2000, 1, 3); let tree = NativeTree::try_new(instant(input(), 60_000, 0, Arc::clone(&out_grid))).unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw(raw_histogram_chunk_with_start( - &schema, - &[2000], - Some(1500), - true, - )) + .push_raw( + InputId::new(0), + raw_histogram_chunk_with_start(&schema, &[2000], Some(1500), true), + ) .unwrap(); let out = run(&mut execution); @@ -3831,27 +3915,19 @@ fn two_series_differing_only_in_an_empty_label_are_both_answered() { let out_grid = grid(0, 60_000, 1); let tree = NativeTree::try_new(instant(input(), 300_000, 0, Arc::clone(&out_grid))).unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // `zone=""` against an absent `zone`: one label set to everything that drops empty values. execution - .push_raw(raw_chunk_of( - &schema, - b"a", - host_metric(Some("")), - &[0], - &[1.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of(&schema, b"a", host_metric(Some("")), &[0], &[1.0], true), + ) .unwrap(); execution - .push_raw(raw_chunk_of( - &schema, - b"b", - host_metric(None), - &[0], - &[2.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of(&schema, b"b", host_metric(None), &[0], &[2.0], true), + ) .unwrap(); let out = run(&mut execution); @@ -3878,13 +3954,13 @@ fn an_instant_selection_reserves_the_series_it_buffers() { let out_grid = grid(0, 60_000, 4); let tree = NativeTree::try_new(instant(input(), 300_000, 0, Arc::clone(&out_grid))).unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); let mut previous = 0; for timestamps in [vec![0, 60_000], vec![120_000], vec![180_000]] { let values = vec![1.0; timestamps.len()]; let chunk = raw_chunk(&schema, ×tamps, &values, false); let bytes = raw_bytes(&chunk); - execution.push_raw(chunk).unwrap(); + execution.push_raw(InputId::new(0), chunk).unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "a series was reported before its last chunk arrived" @@ -3899,7 +3975,10 @@ fn an_instant_selection_reserves_the_series_it_buffers() { previous = held; } execution - .push_raw(raw_chunk(&schema, &[240_000], &[2.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[240_000], &[2.0], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!(out.len(), 1); @@ -3931,10 +4010,10 @@ fn an_instant_selection_reserves_the_workspace_it_gathers() { let out_grid = grid(0, 1, 1_000_000); let tree = NativeTree::try_new(instant(input(), 1_000_000, 0, out_grid)).unwrap(); let pool = bounded_pool(8 * 1024); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let schema = raw_schema(); execution - .push_raw(raw_chunk(&schema, &[0], &[1.0], true)) + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], true)) .unwrap(); let error = execution.next_chunk().unwrap_err(); @@ -4014,8 +4093,8 @@ fn a_selection_reserves_the_points_it_decides() { 5, "the clipping kernel decides another count" ); - let mut execution = executor(&tree, bounded_pool(budget), &ExecutionPlanMetricsSet::new()); - execution.push_raw(chunk).unwrap(); + let mut execution = executor(&tree, bounded_pool(budget)); + execution.push_raw(InputId::new(0), chunk).unwrap(); let out = run(&mut execution); assert_eq!(out.len(), 1); let expected: Vec<(usize, Option)> = timestamps @@ -4039,9 +4118,12 @@ fn a_selection_reserves_the_points_it_decides() { let tree = NativeTree::try_new(instant(input(), 604_800_001, 0, Arc::clone(&out_grid))).unwrap(); let pool = bounded_pool(budget); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_raw(raw_chunk(&schema, &[0, 604_800_000], &[1.0, 2.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 604_800_000], &[1.0, 2.0], true), + ) .unwrap(); let error = execution.next_chunk().unwrap_err(); assert!( @@ -4066,10 +4148,13 @@ fn an_oversized_selection_is_refused_by_the_pool() { let out_grid = grid(0, 1, 1_000_002); let tree = NativeTree::try_new(instant(input(), 1_000_000, 0, out_grid)).unwrap(); let pool = bounded_pool(8 * 1024); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let schema = raw_schema(); execution - .push_raw(raw_chunk(&schema, &[0, 1_000_001], &[1.0, 2.0], true)) + .push_raw( + InputId::new(0), + raw_chunk(&schema, &[0, 1_000_001], &[1.0, 2.0], true), + ) .unwrap(); let error = execution.next_chunk().unwrap_err(); @@ -4101,10 +4186,10 @@ fn an_oversized_selection_is_refused_by_the_pool() { fn an_input_that_ends_inside_a_series_is_reported() { let tree = NativeTree::try_new(instant(input(), 300_000, 0, grid(0, 60_000, 2))).unwrap(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let schema = raw_schema(); execution - .push_raw(raw_chunk(&schema, &[0], &[1.0], false)) + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); @@ -4130,21 +4215,17 @@ fn an_input_that_ends_inside_a_series_is_reported() { fn a_chunk_of_another_series_is_refused() { let tree = NativeTree::try_new(instant(input(), 300_000, 0, grid(0, 60_000, 2))).unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw(raw_chunk(&schema, &[0], &[1.0], false)) + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); // The second chunk carries another series' key. execution - .push_raw(raw_chunk_of( - &schema, - b"series-b", - metric(), - &[60_000], - &[2.0], - true, - )) + .push_raw( + InputId::new(0), + raw_chunk_of(&schema, b"series-b", metric(), &[60_000], &[2.0], true), + ) .unwrap(); let error = execution.next_chunk().unwrap_err(); @@ -4344,7 +4425,7 @@ fn one_metric(job: &str, instance: &str) -> MetricName { /// A tree of two evaluated sources over `points` points under a binary node, with the two paths a /// caller queues at. -fn binary_tree(op: BinaryOp, points: u64) -> (NativeTree, Arc, NodePath, NodePath) { +fn binary_tree(op: BinaryOp, points: u64) -> (NativeTree, Arc, InputId, InputId) { let grid = grid(0, 60_000, points); let root = binary_of( evaluated_input_of(1, Arc::clone(&grid)), @@ -4354,8 +4435,8 @@ fn binary_tree(op: BinaryOp, points: u64) -> (NativeTree, Arc, NodePat ( NativeTree::try_new(root).unwrap(), grid, - NodePath::root().left(), - NodePath::root().right(), + InputId::new(0), + InputId::new(1), ) } @@ -4364,16 +4445,16 @@ fn binary_tree(op: BinaryOp, points: u64) -> (NativeTree, Arc, NodePat fn a_binary_of_two_inputs_adds_the_matched_series() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(grid); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0], true), ) .unwrap(); execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_chunk(&schema, 0, vec![10.0, 20.0, 30.0, 40.0], true), ) .unwrap(); @@ -4392,33 +4473,33 @@ fn a_binary_of_two_inputs_adds_the_matched_series() { fn a_binary_result_does_not_depend_on_arrival_order_or_chunk_boundaries() { let (tree, grid, left, right) = binary_tree(BinaryOp::Sub, 4); let schema = evaluated_schema(Arc::clone(&grid)); - let mut split = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut split = executor(&tree, pool()); split - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0], true), ) .unwrap(); split - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], false)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], false)) .unwrap(); split - .push_evaluated_at(&right, evaluated_chunk(&schema, 2, vec![30.0, 40.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 2, vec![30.0, 40.0], true)) .unwrap(); let split = decided(&run(&mut split)); - let mut reversed = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut reversed = executor(&tree, pool()); reversed - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], false)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], false)) .unwrap(); reversed - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0], true), ) .unwrap(); reversed - .push_evaluated_at(&right, evaluated_chunk(&schema, 2, vec![30.0, 40.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 2, vec![30.0, 40.0], true)) .unwrap(); let reversed = decided(&run(&mut reversed)); @@ -4443,17 +4524,17 @@ fn a_cross_chunk_duplicate_on_one_side_is_refused() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(grid); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); // The same label set twice on the right, each delivered in the chunk that closes it, and both // holding values at positions 0 and 1. execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![30.0, 40.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![30.0, 40.0], true)) .unwrap(); let error = execution.finish().unwrap_err(); @@ -4472,16 +4553,16 @@ fn a_cross_chunk_duplicate_on_one_side_is_refused() { fn an_empty_side_yields_no_series() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // A closing chunk with no row: the series ends before its first point. execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk_of(&schema, metric(), 0, Vec::new(), true), ) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); assert!(run(&mut execution).is_empty()); } @@ -4491,19 +4572,19 @@ fn an_empty_side_yields_no_series() { fn an_empty_closing_chunk_does_not_rewind_a_series() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0], false)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0], false)) .unwrap(); // The chunk that closes the series holds no row, so where it addresses says nothing. execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk_of(&schema, metric(), 0, Vec::new(), true), ) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) .unwrap(); assert_eq!(decided(&run(&mut execution)), vec![(0, Some(11.0))]); } @@ -4513,17 +4594,17 @@ fn an_empty_closing_chunk_does_not_rewind_a_series() { fn a_sparse_binary_keeps_the_shared_positions() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(grid); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // The left side decides positions 0 and 2 only, in two chunks. execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0], false)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0], false)) .unwrap(); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 2, vec![3.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 2, vec![3.0], true)) .unwrap(); execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_chunk(&schema, 0, vec![10.0, 20.0, 30.0, 40.0], true), ) .unwrap(); @@ -4547,11 +4628,11 @@ fn a_binary_pairs_the_fields_of_each_operand() { fields: field_count(2), eval_grid: Some(grid), }); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); execution - .push_evaluated_at( - &NodePath::root().left(), + .push_evaluated( + InputId::new(0), evaluated_columns( &schema, metric(), @@ -4562,8 +4643,8 @@ fn a_binary_pairs_the_fields_of_each_operand() { ) .unwrap(); execution - .push_evaluated_at( - &NodePath::root().right(), + .push_evaluated( + InputId::new(1), evaluated_columns( &schema, metric(), @@ -4580,12 +4661,11 @@ fn a_binary_pairs_the_fields_of_each_operand() { assert_eq!(field_values(&out[0], 1), vec![Some(12.0), Some(14.0)]); // The result decided two positions and holds three values: the two positions are what the node // evaluated, and a value is what a field sample holds, which is one fewer for the null. - let root_path = NodePath::root(); assert_eq!( - count(&set, &root_path, NodeMetric::EVALUATED_GRID_POINTS), + count(&metrics, "root", NodeMetric::EVALUATED_GRID_POINTS), 2 ); - assert_eq!(count(&set, &root_path, NodeMetric::OUTPUT_VALUES), 3); + assert_eq!(count(&metrics, "root", NodeMetric::OUTPUT_VALUES), 3); } /// A comparison without `bool` filters the points it does not hold at; with `bool` it reports both. @@ -4618,16 +4698,16 @@ fn a_comparison_filters_and_a_bool_comparison_reports_both_outcomes() { matching, ); let tree = NativeTree::try_new(root).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated_at( - &NodePath::root().left(), + .push_evaluated( + InputId::new(0), evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0], true), ) .unwrap(); execution - .push_evaluated_at( - &NodePath::root().right(), + .push_evaluated( + InputId::new(1), evaluated_chunk(&schema, 0, vec![2.0, 2.0, 2.0, 2.0], true), ) .unwrap(); @@ -4657,14 +4737,14 @@ fn a_group_modifier_copies_the_one_side_label() { ); let tree = NativeTree::try_new(root).unwrap(); let schema = evaluated_schema(Arc::clone(&grid)); - let (left, right) = (NodePath::root().left(), NodePath::root().right()); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let (left, right) = (InputId::new(0), InputId::new(1)); + let mut execution = executor(&tree, pool()); for (metric, values) in [ (pod_metric("a", "api", "p1"), vec![Some(1.0), Some(2.0)]), (pod_metric("b", "db", "p2"), vec![Some(3.0), Some(4.0)]), ] { execution - .push_evaluated_at(&left, evaluated_chunk_of(&schema, metric, 0, values, true)) + .push_evaluated(left, evaluated_chunk_of(&schema, metric, 0, values, true)) .unwrap(); } for (metric, values) in [ @@ -4672,7 +4752,7 @@ fn a_group_modifier_copies_the_one_side_label() { (one_metric("db", "i2"), vec![Some(100.0), Some(200.0)]), ] { execution - .push_evaluated_at(&right, evaluated_chunk_of(&schema, metric, 0, values, true)) + .push_evaluated(right, evaluated_chunk_of(&schema, metric, 0, values, true)) .unwrap(); } @@ -4721,16 +4801,13 @@ fn a_group_modifier_copies_the_one_side_label() { fn a_binary_adds_the_native_histograms_of_both_sides() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated_at( - &left, - evaluated_histograms(&schema, metric(), 0, 4, 2, true), - ) + .push_evaluated(left, evaluated_histograms(&schema, metric(), 0, 4, 2, true)) .unwrap(); execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_histograms(&schema, metric(), 0, 4, 2, true), ) .unwrap(); @@ -4745,12 +4822,12 @@ fn a_binary_adds_the_native_histograms_of_both_sides() { fn a_binary_yields_nothing_until_the_input_finishes() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) .unwrap(); assert!( execution.next_chunk().unwrap().is_none(), @@ -4771,17 +4848,17 @@ fn a_truncated_operand_is_refused_before_the_kernel_runs() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(grid); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); // The left side holds a duplicate the kernel would report if it were reached. execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![3.0, 4.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![3.0, 4.0], true)) .unwrap(); // The right side never hands over the chunk that closes its series. execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0], false)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0], false)) .unwrap(); let error = execution.finish().unwrap_err(); @@ -4801,12 +4878,12 @@ fn an_oversized_binary_workspace_is_refused_by_the_pool() { let schema = evaluated_schema(grid); // Enough for the two one-row operands, far less than the presence arrays the kernel builds. let pool = bounded_pool(60_000); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0], true)) .unwrap(); let error = execution.finish().unwrap_err(); assert!( @@ -4828,13 +4905,13 @@ fn an_oversized_binary_operand_is_refused_by_the_pool() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 100_000); let schema = evaluated_schema(grid); let pool = bounded_pool(60_000); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0], true)) .unwrap(); let values: Vec = (0..50_000).map(f64::from).collect(); let error = execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, values, true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, values, true)) .unwrap_err(); assert!( matches!(error, DataFusionError::ResourcesExhausted(_)), @@ -5177,23 +5254,23 @@ fn a_binary_beneath_the_executor_reserves_what_it_retains() { .map(|chunk| operand_chunk_bytes(&chunk.series)) .fold(0usize, usize::saturating_add); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, bounded_pool(1 << 22), &set); - execution.push_evaluated_at(&left, first).unwrap(); - execution.push_evaluated_at(&left, second).unwrap(); + let mut execution = executor(&tree, bounded_pool(1 << 22)); + let metrics = execution.state().runtime_tree(); + execution.push_evaluated(left, first).unwrap(); + execution.push_evaluated(left, second).unwrap(); // Nothing is queued on the right, so the pull retains both chunks of the left operand and // nothing else: the kernel has not run and the node holds what it retains. assert!(execution.next_chunk().unwrap().is_none()); - let peak = NodePath::root().partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 0); + let peak = format!("native.p0.root.{}", NodeMetric::PEAK_RESERVED_BYTES); assert_eq!( - gauge(&set, &peak), + gauge(&metrics, &peak), retained, "the binary node reserved more than the operands it retains" ); // The right operand matches the label set, so the operation answers every point of the grid. execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![2.0; 4_000], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![2.0; 4_000], true)) .unwrap(); assert_eq!( addressed(&run(&mut execution)), @@ -5214,12 +5291,12 @@ fn an_ended_input_streams_the_binary_answer_one_chunk_at_a_time() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(Arc::clone(&grid)); let pool = bounded_pool(1 << 22); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0; 4], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0; 4], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![2.0; 4], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![2.0; 4], true)) .unwrap(); // The operands are whole now, but the kernel has not run: the caller has not ended the input. assert!(execution.next_chunk().unwrap().is_none()); @@ -5248,12 +5325,12 @@ fn finish_collects_what_the_streaming_contract_would_have_pulled() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(Arc::clone(&grid)); let pool = bounded_pool(1 << 22); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0; 4], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0; 4], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![2.0; 4], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![2.0; 4], true)) .unwrap(); let chunks = execution.finish().expect("the whole answer is collected"); assert_eq!(addressed(&chunks), vec![(0, vec![Some(3.0); 4])]); @@ -5289,18 +5366,18 @@ fn a_binary_reads_an_operand_a_collecting_window_flushes() { binary_matching(BinaryOp::Add), ); let tree = NativeTree::try_new(root).expect("the shape is legal"); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // One point on each side, in one window: the left is -1, which `abs` makes 1, and the right is // 10, so the sum is 11. execution - .push_evaluated_at( - &NodePath::root().left().input().input(), + .push_evaluated( + InputId::new(0), evaluated_chunk_of(&schema, metric(), 0, vec![Some(-1.0)], true), ) .unwrap(); execution - .push_evaluated_at( - &NodePath::root().right(), + .push_evaluated( + InputId::new(1), evaluated_chunk_of(&schema, metric(), 0, vec![Some(10.0)], true), ) .unwrap(); @@ -5341,14 +5418,10 @@ fn a_series_of_many_chunks_is_one_candidate_for_the_kernel() { .collect(); reset_visited_pairs(); - let mut split = executor( - &tree, - bounded_pool(1 << 26), - &ExecutionPlanMetricsSet::new(), - ); + let mut split = executor(&tree, bounded_pool(1 << 26)); for (position, chunk) in per_point.iter().enumerate() { - split.push_evaluated_at(&left, chunk.clone()).unwrap(); - split.push_evaluated_at(&right, chunk.clone()).unwrap(); + split.push_evaluated(left, chunk.clone()).unwrap(); + split.push_evaluated(right, chunk.clone()).unwrap(); assert!( split.next_chunk().unwrap().is_none(), "the binary node answered at position {position} before its input ended" @@ -5363,14 +5436,10 @@ fn a_series_of_many_chunks_is_one_candidate_for_the_kernel() { assert_eq!(split.len(), 1, "one matched label set answers one chunk"); reset_visited_pairs(); - let mut whole = executor( - &tree, - bounded_pool(1 << 26), - &ExecutionPlanMetricsSet::new(), - ); + let mut whole = executor(&tree, bounded_pool(1 << 26)); let chunk = evaluated_array_chunk_of(&schema, metric(), 0, batch.clone(), true); - whole.push_evaluated_at(&left, chunk.clone()).unwrap(); - whole.push_evaluated_at(&right, chunk).unwrap(); + whole.push_evaluated(left, chunk.clone()).unwrap(); + whole.push_evaluated(right, chunk).unwrap(); assert!(whole.next_chunk().unwrap().is_none()); let whole = whole.finish().unwrap(); assert_eq!( @@ -5473,10 +5542,10 @@ fn an_assembly_the_executors_pool_cannot_hold_stops_it() { ); let (retained, assembly) = binary_charge(&schema, &left_chunk, &right_chunk); let pool = bounded_pool(retained + assembly - 1); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, Arc::clone(&pool), &set); - execution.push_evaluated_at(&left, left_chunk).unwrap(); - execution.push_evaluated_at(&right, right_chunk).unwrap(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = execution.state().runtime_tree(); + execution.push_evaluated(left, left_chunk).unwrap(); + execution.push_evaluated(right, right_chunk).unwrap(); // Both operands are retained before anything is assembled, and nothing is answered: the refusal // below is the assembly's and not a chunk's. assert!(execution.next_chunk().unwrap().is_none()); @@ -5487,8 +5556,8 @@ fn an_assembly_the_executors_pool_cannot_hold_stops_it() { ); assert_eq!( gauge( - &set, - &NodePath::root().partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 0) + &metrics, + &format!("native.p0.root.{}", NodeMetric::PEAK_RESERVED_BYTES) ), retained, "the binary node reserved past what it retains" @@ -5775,13 +5844,13 @@ fn a_mixed_nested_operand_assembles_into_one_series_with_its_holes() { binary_matching(BinaryOp::Add), ); let tree = NativeTree::try_new(root).unwrap(); - let (left, right) = (NodePath::root().left(), NodePath::root().right()); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let (left, right) = (InputId::new(0), InputId::new(1)); + let mut execution = executor(&tree, Arc::clone(&pool)); for chunk in [left_a, left_b] { - execution.push_evaluated_at(&left, chunk).unwrap(); + execution.push_evaluated(left, chunk).unwrap(); } for chunk in [right_a, right_b] { - execution.push_evaluated_at(&right, chunk).unwrap(); + execution.push_evaluated(right, chunk).unwrap(); } let out = run(&mut execution); assert_eq!(out.len(), 1, "one matched label set answers one chunk"); @@ -5803,16 +5872,16 @@ fn a_kernel_error_and_a_drop_release_the_binary_reservation() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); // The same label set twice on the right, both holding a value at position 0. execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![1.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![1.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![2.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![2.0], true)) .unwrap(); let error = execution.finish().unwrap_err(); assert!(error.to_string().contains("duplicate series"), "{error}"); @@ -5820,58 +5889,42 @@ fn a_kernel_error_and_a_drop_release_the_binary_reservation() { assert_eq!(execution.state().reserved_bytes(), 0); // A dropped execution returns its reservation without being told to. - let mut dropped = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut dropped = executor(&tree, Arc::clone(&pool)); dropped - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0], true)) .unwrap(); assert!(pool.reserved() > 0, "the operand reserved nothing"); drop(dropped); assert_eq!(pool.reserved(), 0); } -/// A queued chunk of a two-input tree has to name the input it belongs to. +/// A queued chunk has to name an input the tree has. #[test] -fn a_queued_chunk_names_its_input() { - let (tree, grid, left, _right) = binary_tree(BinaryOp::Add, 2); +fn a_queued_chunk_names_an_input_the_tree_has() { + let (tree, grid, evaluated, _right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); let chunk = evaluated_chunk(&schema, 0, vec![1.0, 2.0], true); - // Unnamed: two inputs make "the input" ambiguous, so nothing is queued at either. - let mut ambiguous = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - let error = ambiguous.push_evaluated(chunk.clone()).unwrap_err(); - assert!(error.to_string().contains("has 2 inputs"), "{error}"); - - // A path that names the binary node rather than an input. - let mut wrong_node = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - let error = wrong_node - .push_evaluated_at(&NodePath::root(), chunk.clone()) - .unwrap_err(); - assert!(error.to_string().contains("only an input node"), "{error}"); - - // A path that names no node at all. - let mut missing = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); - let error = missing - .push_evaluated_at(&NodePath::root().right().right(), chunk) - .unwrap_err(); + // Two leaves are indexed zero and one, so an id past the last one names nothing: the count is + // what says so, rather than an assignment to whichever leaf the count happened to reach. + let mut missing = executor(&tree, pool()); + let error = missing.push_evaluated(InputId::new(2), chunk).unwrap_err(); assert!( - error - .to_string() - .contains("has no node at root.right.right"), + error.to_string().contains("has 2 inputs, and no input 2"), "{error}" ); // An input bound to evaluated samples refuses raw ones, naming the input it was queued at. - let mut fresh = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut fresh = executor(&tree, pool()); let error = fresh - .push_raw_at(&left, raw_chunk(&raw_schema(), &[0], &[1.0], true)) + .push_raw(evaluated, raw_chunk(&raw_schema(), &[0], &[1.0], true)) .unwrap_err(); assert!( - error.to_string().contains(&format!( - "the native source at {left} is bound to an evaluated series" - )), + error + .to_string() + .contains("is bound to an evaluated series"), "{error}" ); - assert_eq!(left, NodePath::root().left()); } /// A map above a binary, and a window above a binary, pull its result through their own loops. @@ -5883,22 +5936,19 @@ fn a_binary_beneath_a_map_and_a_window_is_pulled_recursively() { // The map and the window below are the outermost nodes, so the binary node is one step down // from the root and its two inputs one step below that: `root.input.left` and // `root.input.right`, not `root.left` and `root.right`. - let (left, right) = ( - NodePath::root().input().left(), - NodePath::root().input().right(), - ); + let (left, right) = (InputId::new(0), InputId::new(1)); let root = binary_of( evaluated_input_of(1, Arc::clone(&inner_grid)), evaluated_input_of(1, Arc::clone(&inner_grid)), binary_matching(BinaryOp::Sub), ); let mapped = NativeTree::try_new(map(root, vec![MathOp::Abs])).unwrap(); - let mut execution = executor(&mapped, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&mapped, pool()); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![4.0, 5.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![4.0, 5.0], true)) .unwrap(); assert_eq!( decided(&run(&mut execution)), @@ -5920,12 +5970,12 @@ fn a_binary_beneath_a_map_and_a_window_is_pulled_recursively() { Arc::clone(&outer), )) .unwrap(); - let mut execution = executor(&windowed, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&windowed, pool()); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) .unwrap(); // The outer window decides each point from the binary result, on its own grid. assert_eq!( @@ -5959,22 +6009,18 @@ fn a_binary_over_two_windows_and_selections_is_executed() { binary_matching(BinaryOp::Add), ); let tree = NativeTree::try_new(root).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (path, key, values) in [ ( NodePath::root().left().input(), b"series-a".as_slice(), [1.0, 2.0], ), - ( - NodePath::root().right().input(), - b"series-b".as_slice(), - [10.0, 20.0], - ), + (InputId::new(1), b"series-b".as_slice(), [10.0, 20.0]), ] { execution - .push_raw_at( - &path, + .push_raw( + path, raw_chunk_of(&schema, key, metric(), &[0, 60_000], &values, true), ) .unwrap(); @@ -5991,18 +6037,14 @@ fn a_binary_over_two_windows_and_selections_is_executed() { binary_matching(BinaryOp::Add), ); let tree = NativeTree::try_new(root).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (path, key, value) in [ - (NodePath::root().left().input(), b"series-a".as_slice(), 1.0), - ( - NodePath::root().right().input(), - b"series-b".as_slice(), - 10.0, - ), + (InputId::new(0), b"series-a".as_slice(), 1.0), + (InputId::new(1), b"series-b".as_slice(), 10.0), ] { execution - .push_raw_at( - &path, + .push_raw( + path, raw_chunk_of(&schema, key, metric(), &[0], &[value], true), ) .unwrap(); @@ -6037,10 +6079,10 @@ fn a_binary_of_a_selection_and_a_window_matches_prometheus() { )) .unwrap(); let schema = raw_schema(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_raw_at( - &NodePath::root().input().left().input(), + .push_raw( + InputId::new(0), raw_chunk_of( &schema, b"left_raw", @@ -6052,8 +6094,8 @@ fn a_binary_of_a_selection_and_a_window_matches_prometheus() { ) .unwrap(); execution - .push_raw_at( - &NodePath::root().input().right().input(), + .push_raw( + InputId::new(0), raw_chunk_of( &schema, b"right_raw", @@ -6085,22 +6127,22 @@ fn a_chunk_of_another_series_on_one_operand_is_refused() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(grid); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_chunk_of(&schema, named_metric("api", "x"), 0, vec![Some(1.0)], false), ) .unwrap(); // The chunk that arrived while `api` is open names another series. execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0], true)) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_chunk_of(&schema, named_metric("db", "x"), 1, vec![Some(2.0)], true), ) .unwrap(); @@ -6135,18 +6177,18 @@ fn an_operand_chunk_that_rewinds_is_refused_with_or_without_values() { let (tree, grid, left, _right) = binary_tree(BinaryOp::Add, 4); let schema = evaluated_schema(grid); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); // Two positions that hold no value at all, and a series that did not close on them. execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk_of(&schema, metric(), 0, vec![None, None], false), ) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); // Position 1 is one of the two the series already placed. execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 1, vec![1.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 1, vec![1.0], true)) .unwrap(); let error = execution.next_chunk().unwrap_err(); @@ -6210,8 +6252,8 @@ fn a_chunk_that_does_not_match_its_operand_is_refused_before_the_kernel() { ), ]; for (case, chunk) in cases { - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); - execution.push_evaluated_at(&left, chunk).unwrap(); + let mut execution = executor(&tree, Arc::clone(&pool)); + execution.push_evaluated(left, chunk).unwrap(); let error = execution.next_chunk().unwrap_err(); assert!( error.to_string().contains("rejected its left operand"), @@ -6254,15 +6296,15 @@ fn an_absent_operand_yields_no_series() { let pool = pool(); // Neither side queued a chunk. - let mut empty = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut empty = executor(&tree, Arc::clone(&pool)); assert!(empty.finish().unwrap().is_empty()); assert_eq!(pool.reserved(), 0); assert_eq!(empty.state().nodes_holding_binary(), 0); // Only the right side ran, and it closed: nothing matches a series that is not there. - let mut one_side = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut one_side = executor(&tree, Arc::clone(&pool)); one_side - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); assert!(one_side.finish().unwrap().is_empty()); assert_eq!(pool.reserved(), 0); @@ -6275,22 +6317,21 @@ fn an_absent_operand_yields_no_series() { fn a_binary_counts_each_closed_result_once() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2); let schema = evaluated_schema(grid); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); execution - .push_evaluated_at(&left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) + .push_evaluated(left, evaluated_chunk(&schema, 0, vec![1.0, 2.0], true)) .unwrap(); execution - .push_evaluated_at(&right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) + .push_evaluated(right, evaluated_chunk(&schema, 0, vec![10.0, 20.0], true)) .unwrap(); let out = run(&mut execution); assert_eq!(out.len(), 1); assert!(out[0].is_last, "a result series is handed over closed"); - let root = NodePath::root(); // One matched result closed, not one per operand chunk and not one per handover of the result. - assert_eq!(count(&set, &root, NodeMetric::COMPLETED_SERIES), 1); - assert_eq!(count(&set, &root, NodeMetric::INPUT_BATCHES), 2); - assert_eq!(count(&set, &root, NodeMetric::OUTPUT_BATCHES), 1); + assert_eq!(count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 1); + assert_eq!(count(&metrics, "root", NodeMetric::INPUT_BATCHES), 2); + assert_eq!(count(&metrics, "root", NodeMetric::OUTPUT_BATCHES), 1); } /// A legitimate empty result over a wide operand finishes in a 4 MiB pool and holds nothing after. @@ -6303,19 +6344,19 @@ fn an_empty_result_of_a_wide_operand_fits_its_pool() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, points as u64); let schema = evaluated_schema(grid); let pool = bounded_pool(4 << 20); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let series = |job: &str| pod_metric("requests", job, "p"); execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk_of(&schema, series("left"), 0, vec![Some(1.0); points], true), ) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); for index in 0..32 { execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_chunk_of( &schema, series(&format!("r{index:02}")), @@ -6362,20 +6403,20 @@ fn a_dense_wide_result_fits_a_32_mib_pool() { let root = binary_of(left_node, right_node, binary_matching(BinaryOp::Add)); let tree = NativeTree::try_new(root).unwrap(); let pool = bounded_pool(32 << 20); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let columns = |value: f64| -> Vec>> { (0..width).map(|_| vec![Some(value); points]).collect() }; - let (left, right) = (NodePath::root().left(), NodePath::root().right()); + let (left, right) = (InputId::new(0), InputId::new(1)); execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_columns(&schema, metric(), 0, columns(1.0), true), ) .unwrap(); execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_columns(&schema, metric(), 0, columns(2.0), true), ) .unwrap(); @@ -6414,10 +6455,10 @@ fn an_all_missing_match_group_fits_its_pool() { let (tree, grid, left, right) = binary_tree(BinaryOp::Add, points as u64); let schema = evaluated_schema(grid); let pool = bounded_pool(4 << 20); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at( - &left, + .push_evaluated( + left, evaluated_chunk_of(&schema, metric(), 0, vec![Some(1.0); points], true), ) .unwrap(); @@ -6425,8 +6466,8 @@ fn an_all_missing_match_group_fits_its_pool() { // Both right chunks close the one label set, so its match group has two members. for _ in 0..2 { execution - .push_evaluated_at( - &right, + .push_evaluated( + right, evaluated_chunk_of(&schema, metric(), 0, vec![None; points], true), ) .unwrap(); @@ -6531,9 +6572,12 @@ fn a_constant_on_the_right_multiplies_every_value() { false, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![1.0, 2.0, 3.0, 4.0], true), + ) .unwrap(); let streamed = execution .next_chunk() @@ -6672,8 +6716,8 @@ fn an_aggregate_drops_every_label_and_adds_its_group_per_point() { sum_grouping(Grouping::By(Arc::new([]))), )) .unwrap(); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); for (metric, values) in [ ( job_instance_metric("requests", "api", "a"), @@ -6685,7 +6729,10 @@ fn an_aggregate_drops_every_label_and_adds_its_group_per_point() { ), ] { execution - .push_evaluated(evaluated_chunk_of(&schema, metric, 0, values, true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric, 0, values, true), + ) .unwrap(); } @@ -6702,10 +6749,7 @@ fn an_aggregate_drops_every_label_and_adds_its_group_per_point() { ); // One group is one series, so it closes exactly once and its chunk says so. assert!(output.len() == 1 && output[0].is_last); - assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), - 1 - ); + assert_eq!(count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 1); assert_eq!(execution.state().reserved_bytes(), 0); } @@ -6720,9 +6764,9 @@ fn a_broadcast_keeps_the_operand_direction() { let values = || vec![100.0, 200.0, 400.0]; let run_one = |root: NativeNode| -> Vec> { let tree = NativeTree::try_new(root).unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, values(), true)) + .push_evaluated(InputId::new(0), evaluated_chunk(&schema, 0, values(), true)) .unwrap(); let out = run(&mut execution); assert_eq!(out.len(), 1, "one chunk in, one chunk out"); @@ -6782,9 +6826,12 @@ fn a_bool_comparison_reports_one_and_zero() { true, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![1.0, -1.0, 0.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![1.0, -1.0, 0.0], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -6810,9 +6857,12 @@ fn a_filtering_comparison_keeps_the_positions_that_passed() { false, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![1.0, -1.0, 0.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![1.0, -1.0, 0.0], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -6839,9 +6889,12 @@ fn a_constant_of_minus_one_on_the_left_negates_the_vector() { false, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![1.0, -2.0, 5.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![1.0, -2.0, 5.0], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -6866,18 +6919,15 @@ fn a_broadcast_preserves_missing_values_and_skipped_positions() { false, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // Position 0 holds a value and position 1 holds none: the axis covers both, so the difference // is the field's validity rather than the axis's rows. execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 0, - vec![Some(3.0), None], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 0, vec![Some(3.0), None], false), + ) .unwrap(); let first = execution .next_chunk() @@ -6892,13 +6942,10 @@ fn a_broadcast_preserves_missing_values_and_skipped_positions() { // The last chunk sits at position 5: positions 2 to 4 are skipped, not empty values. execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 5, - vec![Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 5, vec![Some(4.0)], true), + ) .unwrap(); let second = execution .next_chunk() @@ -6925,10 +6972,13 @@ fn a_filter_empty_nonclosing_chunk_does_not_end_the_broadcast() { false, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![-1.0, -2.0], false)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![-1.0, -2.0], false), + ) .unwrap(); assert!( execution.next_chunk().unwrap().is_none(), @@ -6936,7 +6986,10 @@ fn a_filter_empty_nonclosing_chunk_does_not_end_the_broadcast() { ); execution - .push_evaluated(evaluated_chunk(&schema, 2, vec![3.0, -4.0, 5.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 2, vec![3.0, -4.0, 5.0], true), + ) .unwrap(); let passing = execution .next_chunk() @@ -7066,16 +7119,22 @@ fn a_filter_that_passes_nothing_still_closes_the_series() { false, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![-1.0, -2.0], false)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![-1.0, -2.0], false), + ) .unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "a chunk that passed nothing and closed nothing has nothing to hand over" ); execution - .push_evaluated(evaluated_chunk(&schema, 2, vec![-3.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 2, vec![-3.0], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -7108,15 +7167,12 @@ fn a_filter_that_passes_nothing_still_closes_the_series() { true, )) .unwrap(); - let mut execution = executor(&bool_tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&bool_tree, pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 0, - vec![None; 3], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 0, vec![None; 3], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -7153,16 +7209,14 @@ fn a_broadcast_does_not_retain_the_vector_it_reads() { )) .unwrap(); let pool = bounded_pool(32 << 10); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let mut peak = 0; for index in 0..count { execution - .push_evaluated(evaluated_chunk( - &schema, - index, - vec![index as f64], - index + 1 == count, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, index, vec![index as f64], index + 1 == count), + ) .unwrap_or_else(|error| { panic!("chunk {index} of {count} did not fit: {error}"); }); @@ -7221,26 +7275,32 @@ fn a_window_above_a_broadcast_merges_the_series_it_collapses() { Arc::clone(&grid), )) .unwrap(); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); // Two series that only differ by name, each holding the half of the grid the other leaves empty. execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("requests", "x"), - 0, - vec![Some(1.0), Some(2.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("requests", "x"), + 0, + vec![Some(1.0), Some(2.0)], + true, + ), + ) .unwrap(); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("requests_total", "x"), - 2, - vec![Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("requests_total", "x"), + 2, + vec![Some(3.0), Some(4.0)], + true, + ), + ) .unwrap(); let out = run(&mut execution); @@ -7263,7 +7323,7 @@ fn a_window_above_a_broadcast_merges_the_series_it_collapses() { // with `is_last`, so two source series ended. What proves the merge happened is the single // closing chunk above -- a window that streamed would have closed each piece on its own. assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), + count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 2, "the window did not see both source series end" ); @@ -7296,16 +7356,13 @@ fn complete_aggregate_kinds_fold_two_values() { }, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for value in [1.0, 3.0] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 0, - vec![Some(value)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 0, vec![Some(value)], true), + ) .unwrap(); } @@ -7337,7 +7394,7 @@ fn an_aggregate_folds_a_series_split_across_chunks_like_a_whole_one() { ); // One chunk per series. - let mut whole = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut whole = executor(&tree(), pool()); for (metric, values) in [ ( first.clone(), @@ -7349,45 +7406,45 @@ fn an_aggregate_folds_a_series_split_across_chunks_like_a_whole_one() { ), ] { whole - .push_evaluated(evaluated_chunk_of(&schema, metric, 0, values, true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric, 0, values, true), + ) .unwrap(); } let whole = whole.finish().unwrap(); // The first series split in two, the second whole: the group is the same group, folded from // positions the chunks name absolutely. - let mut split = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut split = executor(&tree(), pool()); split - .push_evaluated(evaluated_chunk_of( - &schema, - first.clone(), - 0, - vec![Some(1.0), Some(2.0)], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, first.clone(), 0, vec![Some(1.0), Some(2.0)], false), + ) .unwrap(); assert!( split.next_chunk().unwrap().is_none(), "a chunk that does not close its series answered for the group" ); split - .push_evaluated(evaluated_chunk_of( - &schema, - first, - 2, - vec![Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, first, 2, vec![Some(3.0), Some(4.0)], true), + ) .unwrap(); split - .push_evaluated(evaluated_chunk_of( - &schema, - second, - 0, - vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + second, + 0, + vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)], + true, + ), + ) .unwrap(); let split = split.finish().unwrap(); @@ -7421,25 +7478,31 @@ fn an_aggregate_reports_only_the_points_a_group_holds_a_value_at() { sum_grouping(Grouping::By(Arc::new([]))), )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); // Positions 0 and 1 of the first series, 1 and 2 of the second, and nothing at 3. execution - .push_evaluated(sparse_chunk( - &schema, - job_instance_metric("requests", "api", "a"), - &[(0, 2)], - vec![1.0, 2.0], - true, - )) + .push_evaluated( + InputId::new(0), + sparse_chunk( + &schema, + job_instance_metric("requests", "api", "a"), + &[(0, 2)], + vec![1.0, 2.0], + true, + ), + ) .unwrap(); execution - .push_evaluated(sparse_chunk( - &schema, - job_instance_metric("requests", "db", "b"), - &[(1, 2)], - vec![20.0, 300.0], - true, - )) + .push_evaluated( + InputId::new(0), + sparse_chunk( + &schema, + job_instance_metric("requests", "db", "b"), + &[(1, 2)], + vec![20.0, 300.0], + true, + ), + ) .unwrap(); let output = execution.finish().unwrap(); @@ -7462,7 +7525,7 @@ fn an_aggregate_over_no_series_produces_nothing() { sum_grouping(Grouping::By(Arc::new(["job".into()]))), )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); let output = execution.finish().unwrap(); assert!( output.is_empty(), @@ -7487,13 +7550,19 @@ fn a_broadcast_reports_one_chunk_in_for_each_chunk_out() { false, )) .unwrap(); - let set = ExecutionPlanMetricsSet::new(); - let mut execution = executor(&tree, pool(), &set); + let mut execution = executor(&tree, pool()); + let metrics = execution.state().runtime_tree(); execution - .push_evaluated(evaluated_chunk(&schema, 0, vec![1.0, 2.0], false)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 0, vec![1.0, 2.0], false), + ) .unwrap(); execution - .push_evaluated(evaluated_chunk(&schema, 2, vec![3.0, 4.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&schema, 2, vec![3.0, 4.0], true), + ) .unwrap(); let out = run(&mut execution); assert_eq!( @@ -7505,23 +7574,23 @@ fn a_broadcast_reports_one_chunk_in_for_each_chunk_out() { "each chunk is answered as it arrived, at the positions it held" ); assert_eq!( - count(&set, &NodePath::root(), NodeMetric::INPUT_BATCHES), + count(&metrics, "root", NodeMetric::INPUT_BATCHES), 2, "the node read two chunks" ); assert_eq!( - count(&set, &NodePath::root(), NodeMetric::OUTPUT_BATCHES), + count(&metrics, "root", NodeMetric::OUTPUT_BATCHES), 2, "the node handed two chunks over" ); assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), + count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 1, "the series closed once, on the chunk that said so" ); // The scalar side is a node of the tree and reads nothing: it has no input at all. assert_eq!( - count(&set, &NodePath::root().right(), NodeMetric::INPUT_BATCHES), + count(&metrics, "root.right", NodeMetric::INPUT_BATCHES), 0, "the constant was pulled as a stream" ); @@ -7546,16 +7615,14 @@ fn a_broadcast_over_a_streaming_window_hands_over_the_points_it_settles() { false, )) .unwrap(); - let set = ExecutionPlanMetricsSet::new(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &set); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = execution.state().runtime_tree(); execution - .push_evaluated(evaluated_chunk( - &inner_schema, - 0, - vec![1.0, 2.0, 3.0], - false, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&inner_schema, 0, vec![1.0, 2.0, 3.0], false), + ) .unwrap(); let mut out = Vec::new(); @@ -7564,13 +7631,13 @@ fn a_broadcast_over_a_streaming_window_hands_over_the_points_it_settles() { } assert_eq!(decided(&out), vec![(0, Some(100.0)), (1, Some(300.0))]); assert!(out.iter().all(|chunk| !chunk.is_last)); - assert_eq!( - count(&set, &NodePath::root(), NodeMetric::COMPLETED_SERIES), - 0 - ); + assert_eq!(count(&metrics, "root", NodeMetric::COMPLETED_SERIES), 0); execution - .push_evaluated(evaluated_chunk(&inner_schema, 3, vec![4.0, 5.0], true)) + .push_evaluated( + InputId::new(0), + evaluated_chunk(&inner_schema, 3, vec![4.0, 5.0], true), + ) .unwrap(); while let Some(chunk) = execution.next_chunk().unwrap() { out.push(chunk); @@ -7606,10 +7673,10 @@ fn a_broadcast_above_a_collecting_window_waits_for_its_flush() { )) .unwrap(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated_at( - &NodePath::root().left().input().input(), + .push_evaluated( + InputId::new(0), evaluated_chunk_of(&schema, metric(), 0, vec![Some(-1.0)], true), ) .unwrap(); @@ -7635,17 +7702,14 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv )) .unwrap(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let first_metric = MetricName::try_new(Some("requests".to_string().into()), Vec::new()).unwrap(); execution - .push_evaluated(evaluated_chunk_of( - &schema, - first_metric, - 0, - vec![Some(1.0), Some(2.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, first_metric, 0, vec![Some(1.0), Some(2.0)], true), + ) .unwrap(); let first = execution .next_chunk() @@ -7659,13 +7723,10 @@ fn a_broadcast_refuses_two_series_that_share_a_label_set_and_releases_the_reserv let second_metric = MetricName::try_new(Some("requests_total".to_string().into()), Vec::new()).unwrap(); execution - .push_evaluated(evaluated_chunk_of( - &schema, - second_metric, - 0, - vec![Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, second_metric, 0, vec![Some(3.0), Some(4.0)], true), + ) .unwrap(); let error = execution .next_chunk() @@ -7701,7 +7762,7 @@ fn an_aggregate_answers_nothing_until_its_whole_input_ends() { sum_grouping(Grouping::By(Arc::new([]))), )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); let computed = |execution: &NativeExecutor| { execution.state().runtimes[0] .aggregate @@ -7711,13 +7772,16 @@ fn an_aggregate_answers_nothing_until_its_whole_input_ends() { }; execution - .push_evaluated(evaluated_chunk_of( - &schema, - job_instance_metric("requests", "api", "a"), - 0, - vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + job_instance_metric("requests", "api", "a"), + 0, + vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)], + true, + ), + ) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); assert!(!computed(&execution), "the fold finished on one series"); @@ -7727,13 +7791,16 @@ fn an_aggregate_answers_nothing_until_its_whole_input_ends() { ); // The second series closed too, and the input still has not ended. execution - .push_evaluated(evaluated_chunk_of( - &schema, - job_instance_metric("requests", "db", "b"), - 0, - vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + job_instance_metric("requests", "db", "b"), + 0, + vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)], + true, + ), + ) .unwrap(); assert!( !computed(&execution), @@ -7780,15 +7847,18 @@ fn an_aggregate_refuses_a_chunk_of_another_layout_and_returns_its_reservation() sum_grouping(Grouping::By(Arc::new([]))), )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 0, - vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + metric(), + 0, + vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)], + true, + ), + ) .unwrap(); // Two fields where the tree bound one: the source node accepts it, the fold refuses it. let wider = Arc::new(SeriesSchema { @@ -7796,13 +7866,16 @@ fn an_aggregate_refuses_a_chunk_of_another_layout_and_returns_its_reservation() eval_grid: Some(Arc::clone(&grid)), }); execution - .push_evaluated(evaluated_columns( - &wider, - metric(), - 0, - vec![vec![Some(1.0); 4], vec![Some(2.0); 4]], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_columns( + &wider, + metric(), + 0, + vec![vec![Some(1.0); 4], vec![Some(2.0); 4]], + true, + ), + ) .unwrap(); let error = execution @@ -7837,15 +7910,12 @@ fn aggregate_budget_rejects_first_block_and_releases_everything() { )) .unwrap(); let pool = bounded_pool(2_000); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 0, - vec![Some(2.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 0, vec![Some(2.0)], true), + ) .unwrap(); assert!(execution.next_chunk().is_err()); assert_eq!(pool.reserved(), 0); @@ -7864,16 +7934,19 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() { )) .unwrap(); let pool = bounded_pool(1 << 20); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); for (job, value) in [("api", 1.0), ("db", 2.0)] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - job_instance_metric("requests", job, "a"), - 0, - vec![Some(value)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + job_instance_metric("requests", job, "a"), + 0, + vec![Some(value)], + true, + ), + ) .unwrap(); } execution.end_of_input().unwrap(); @@ -7918,15 +7991,12 @@ fn aggregate_sparse_output_budget_uses_rows_not_grid_length() { )) .unwrap(); let pool = bounded_pool(64 * 1024); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 10, - vec![Some(2.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 10, vec![Some(2.0)], true), + ) .unwrap(); let output = execution.finish().unwrap(); assert_eq!(groups_of(&output), vec![unlabelled(vec![(10, Some(2.0))])]); @@ -7944,15 +8014,12 @@ fn aggregate_fold_state_is_released_after_input() { )) .unwrap(); let pool = bounded_pool(1 << 20); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); execution - .push_evaluated(evaluated_chunk_of( - &schema, - metric(), - 0, - vec![Some(2.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, metric(), 0, vec![Some(2.0)], true), + ) .unwrap(); assert!(execution.next_chunk().unwrap().is_none()); let runtime = &execution.state.runtimes[0]; @@ -8000,12 +8067,7 @@ fn an_aggregate_kind_or_stage_the_runtime_does_not_compute_is_named() { }; for (found, tree) in [("Quantile", kind_tree), ("Partial", partial_tree)] { - let error = match NativeExecutionState::try_new( - &tree, - pool(), - &ExecutionPlanMetricsSet::new(), - 0, - ) { + let error = match NativeExecutionState::try_new(&tree, pool()) { Ok(_) => panic!("the runtime accepted a {found} aggregate"), Err(error) => error, }; @@ -8052,30 +8114,36 @@ fn an_aggregate_above_a_collecting_window_waits_for_the_merge() { .unwrap() }; - let mut execution = executor(&tree(), pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree(), pool()); // Two series that only differ by name, each holding half of the grid the other leaves empty: // after the name is dropped they are one series only once merged. execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("requests", "x"), - 0, - vec![Some(1.0), Some(2.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("requests", "x"), + 0, + vec![Some(1.0), Some(2.0)], + true, + ), + ) .unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "the aggregate answered before the window above it had merged" ); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("requests_total", "x"), - 2, - vec![Some(3.0), Some(4.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("requests_total", "x"), + 2, + vec![Some(3.0), Some(4.0)], + true, + ), + ) .unwrap(); let out = run(&mut execution); @@ -8140,7 +8208,7 @@ fn topk_ranks_whole_series_at_each_point() { )) .unwrap(); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); for chunk in [ evaluated_chunk_of( &schema, @@ -8158,7 +8226,7 @@ fn topk_ranks_whole_series_at_each_point() { true, ), ] { - execution.push_evaluated(chunk).unwrap(); + execution.push_evaluated(InputId::new(0), chunk).unwrap(); } let chunks = run(&mut execution); assert_eq!( @@ -8188,7 +8256,7 @@ fn limitk_takes_series_in_input_order() { }, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for chunk in [ evaluated_chunk_of(&schema, named_metric("a", "z"), 1, vec![Some(1.0)], true), evaluated_chunk_of( @@ -8200,7 +8268,7 @@ fn limitk_takes_series_in_input_order() { ), evaluated_chunk_of(&schema, named_metric("c", "z"), 0, vec![Some(3.0)], true), ] { - execution.push_evaluated(chunk).unwrap(); + execution.push_evaluated(InputId::new(0), chunk).unwrap(); } let chunks = run(&mut execution); assert_eq!( @@ -8229,8 +8297,8 @@ fn a_binary_node_reads_what_a_vector_node_kept() { binary_matching(BinaryOp::Add), )) .unwrap(); - let (left, right) = (NodePath::root().left().input(), NodePath::root().right()); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let (left, right) = (InputId::new(0), InputId::new(1)); + let mut execution = executor(&tree, pool()); for (path, chunk) in [ ( &left, @@ -8263,7 +8331,7 @@ fn a_binary_node_reads_what_a_vector_node_kept() { ), ), ] { - execution.push_evaluated_at(path, chunk).unwrap(); + execution.push_evaluated(path, chunk).unwrap(); } assert_eq!(decided(&run(&mut execution)), vec![(1, Some(15.0))]); } @@ -8291,22 +8359,19 @@ fn absent_answers_where_the_input_holds_nothing() { )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("up", "y"), - 1, - vec![Some(0.0)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric("up", "y"), 1, vec![Some(0.0)], true), + ) .unwrap(); let chunks = run(&mut execution); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].series.metric, labels); assert_eq!(decided(&chunks), vec![(0, Some(1.0)), (2, Some(1.0))]); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); assert_eq!( decided(&run(&mut execution)), vec![(0, Some(1.0)), (1, Some(1.0)), (2, Some(1.0))] @@ -8323,16 +8388,13 @@ fn sort_desc_hands_over_series_in_value_order() { BoundVectorOp::Sort(SortOrder::Desc), )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (job, value) in [("b", 1.0), ("a", 3.0), ("c", 2.0)] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("m", job), - 0, - vec![Some(value)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric("m", job), 0, vec![Some(value)], true), + ) .unwrap(); } let jobs: Vec = run(&mut execution) @@ -8356,17 +8418,20 @@ fn a_sorted_series_is_queued_at_what_its_slices_hold() { )) .unwrap(); let pool = bounded_pool(48 * 1024); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); let values = Float64Array::from((0..points).map(|v| v as f64).collect::>()); for start in (0..points as usize).step_by(100) { execution - .push_evaluated(evaluated_array_chunk_of( - &schema, - named_metric("m", "a"), - start, - values.slice(start, 100), - start + 100 == points as usize, - )) + .push_evaluated( + InputId::new(0), + evaluated_array_chunk_of( + &schema, + named_metric("m", "a"), + start, + values.slice(start, 100), + start + 100 == points as usize, + ), + ) .unwrap(); // The vector node retains each chunk as it arrives, at what its slice holds. assert!(execution.next_chunk().unwrap().is_none()); @@ -8389,17 +8454,20 @@ fn a_table_shared_by_queued_results_stays_charged_until_the_last_leaves() { .unwrap(); let table = histogram_table(2, 64); let pool = pool(); - let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, Arc::clone(&pool)); for (job, index) in [("a", 0), ("b", 1)] { execution - .push_evaluated(evaluated_histogram_chunk_of( - &schema, - named_metric("m", job), - 0, - vec![Some(index)], - &table, - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of( + &schema, + named_metric("m", job), + 0, + vec![Some(index)], + &table, + true, + ), + ) .unwrap(); } execution.end_of_input().unwrap(); @@ -8430,16 +8498,13 @@ fn limitk_keeps_the_order_a_sort_below_it_decided() { }, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (job, value) in [("a", 1.0), ("z", 2.0)] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("v", job), - 0, - vec![Some(value)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, named_metric("v", job), 0, vec![Some(value)], true), + ) .unwrap(); } assert_eq!( @@ -8475,16 +8540,13 @@ fn histogram_quantile_reads_the_buckets_of_the_whole_vector() { ) .unwrap() }; - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (le, count) in [("+Inf", 2.0), ("1", 1.0)] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - bucket(le), - 0, - vec![Some(count)], - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of(&schema, bucket(le), 0, vec![Some(count)], true), + ) .unwrap(); } let chunks = run(&mut execution); @@ -8520,20 +8582,23 @@ fn histogram_quantile_reads_the_pieces_of_one_label_set_as_one_series() { .unwrap() }; let table = histogram_table(2, 4); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (start, tags, index) in [ (0, vec![("job", "a"), ("zone", "")], 0), (1, vec![("job", "a")], 1), ] { execution - .push_evaluated(evaluated_histogram_chunk_of( - &schema, - metric(&tags), - start, - vec![Some(index)], - &table, - true, - )) + .push_evaluated( + InputId::new(0), + evaluated_histogram_chunk_of( + &schema, + metric(&tags), + start, + vec![Some(index)], + &table, + true, + ), + ) .unwrap(); } let chunks = run(&mut execution); @@ -8558,7 +8623,7 @@ fn count_values_counts_the_samples_sharing_a_value() { }, )) .unwrap(); - let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new()); + let mut execution = executor(&tree, pool()); for (job, start, value, is_last) in [ ("a", 0, 1.0, false), ("a", 1, 1.0, true), @@ -8566,13 +8631,16 @@ fn count_values_counts_the_samples_sharing_a_value() { ("c", 0, 1.0, true), ] { execution - .push_evaluated(evaluated_chunk_of( - &schema, - named_metric("m", job), - start, - vec![Some(value)], - is_last, - )) + .push_evaluated( + InputId::new(0), + evaluated_chunk_of( + &schema, + named_metric("m", job), + start, + vec![Some(value)], + is_last, + ), + ) .unwrap(); } let mut counts: Vec<(String, Decided)> = run(&mut execution) diff --git a/src/promql/src/native/metrics.rs b/src/promql/src/native/metrics.rs index fb0e769d221..a163d292151 100644 --- a/src/promql/src/native/metrics.rs +++ b/src/promql/src/native/metrics.rs @@ -12,105 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Where a node sits in the final execution tree, and the metrics named after that position. +//! Per-node metric handles for native PromQL execution and their temporary DataFusion export. //! -//! A node is identified by its structural position — `root`, `.input`, `.left`, `.right` — rather -//! than by an id it carries. The position is not stored on the node: it is derived while walking -//! the tree. [`walk`] is the one traversal rule, and [`register_tree`] is its caller. -//! -//! The position stays in the metric name because the collector aggregates by name first: two -//! `rate` nodes sharing a bare name would be merged into one number. Positions are only an -//! observation detail of one plan — they are not stable across queries, plan rewrites or versions, -//! so nothing may depend on them beyond correlating a metric with a tree it was taken from. - -use std::sync::Arc; +//! Node metrics retain no names or tree topology. Runtime callers may export their handles into a +//! fresh temporary DataFusion metrics set using names collected for that export. use datafusion::physical_plan::metrics::MetricValue; use datafusion::physical_plan::metrics::{ Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, Time, }; -use crate::native::plan::{NativeExpr, NativeNode}; - /// The reserved first segment of every native metric name. const NATIVE_PREFIX: &str = "native"; -/// A node's position in the final tree. -/// -/// Rendering allocates a short string once, when the run-time state is created; the hot path holds -/// the metric handles instead of looking anything up by name. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct NodePath(String); - -impl NodePath { - /// The position of the root node. - pub fn root() -> Self { - Self("root".to_string()) - } - - /// The position of a unary node's input. - pub fn input(&self) -> Self { - Self(format!("{}.input", self.0)) - } - - /// The position of a binary node's left input. - pub fn left(&self) -> Self { - Self(format!("{}.left", self.0)) - } - - /// The position of a binary node's right input. - pub fn right(&self) -> Self { - Self(format!("{}.right", self.0)) - } - - /// The path without the metric namespace. - pub fn as_str(&self) -> &str { - &self.0 - } - - /// The full metric name for `metric` at this position. - pub fn metric_name(&self, metric: &str) -> String { - format!("{NATIVE_PREFIX}.{}.{}", self.0, metric) - } - - /// The full metric name for `metric` at this position in `partition`. - /// - /// A figure that must not be merged across partitions needs the partition in its name: the - /// collector aggregates by name first, and a sum of peaks is not a peak. - pub fn partitioned_metric_name(&self, metric: &str, partition: usize) -> String { - format!("{NATIVE_PREFIX}.p{partition}.{}.{metric}", self.0) - } -} - -impl std::fmt::Display for NodePath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -/// Visits every node of `root` with its position, outermost first. -/// -/// This is the one traversal rule: [`register_tree`] is its caller, so a node's metric is always -/// named after the position this walk gives it. -pub fn walk(root: &NativeNode, visit: &mut dyn FnMut(&NativeNode, &NodePath)) { - fn go(node: &NativeNode, path: &NodePath, visit: &mut dyn FnMut(&NativeNode, &NodePath)) { - visit(node, path); - match &node.expr { - NativeExpr::Input(_) | NativeExpr::Scalar(_) => {} - NativeExpr::Instant { input, .. } - | NativeExpr::Window { input, .. } - | NativeExpr::Map { input, .. } - | NativeExpr::Aggregate { input, .. } - | NativeExpr::Vector { input, .. } => go(input, &path.input(), visit), - NativeExpr::Binary { left, right, .. } => { - go(left, &path.left(), visit); - go(right, &path.right(), visit); - } - } - } - go(root, &NodePath::root(), visit); -} - /// The metric names one node reports. /// /// [`Self::ALL`] is the list a reader should treat as complete: [`NodeMetrics::register`] names each @@ -196,13 +110,27 @@ impl Default for NodeMetrics { } } +/// The full metric name for `metric` reported by the node called `name`. +fn named(name: &str, metric: &str) -> String { + format!("{NATIVE_PREFIX}.{name}.{metric}") +} + +/// The full metric name for `metric` in `partition` of the node called `name`. +/// +/// A figure that must not be merged across partitions needs the partition in its name: the +/// collector aggregates by name first, and a sum of peaks is not a peak. +fn partitioned(name: &str, metric: &str, partition: usize) -> String { + format!("{NATIVE_PREFIX}.p{partition}.{name}.{metric}") +} + impl NodeMetrics { - /// Registers this node's metrics under `path` in `set`. + /// Exports this node's metrics under its temporary `name` into `set`. /// - /// `partition` is part of the registration, so two partitions of one plan report separately. - /// It is also part of the peak's name, because a peak is the one figure a collector must not - /// add up across partitions. - pub fn register(&self, path: &NodePath, set: &ExecutionPlanMetricsSet, partition: usize) { + /// Callers must invoke this only when collecting a runtime execution into a fresh temporary + /// metrics set. Names belong to that export and are not retained by this node. `partition` is + /// part of the DataFusion registration, and the peak name includes it because peaks must not be + /// summed across partitions. + pub fn register(&self, name: &str, set: &ExecutionPlanMetricsSet, partition: usize) { fn count(set: &ExecutionPlanMetricsSet, partition: usize, name: String, count: &Count) { MetricBuilder::new(set) .with_partition(partition) @@ -223,43 +151,43 @@ impl NodeMetrics { count( set, partition, - path.metric_name(NodeMetric::INPUT_BATCHES), + named(name, NodeMetric::INPUT_BATCHES), &self.input_batches, ); count( set, partition, - path.metric_name(NodeMetric::OUTPUT_BATCHES), + named(name, NodeMetric::OUTPUT_BATCHES), &self.output_batches, ); count( set, partition, - path.metric_name(NodeMetric::INPUT_SAMPLES), + named(name, NodeMetric::INPUT_SAMPLES), &self.input_samples, ); count( set, partition, - path.metric_name(NodeMetric::EVALUATED_GRID_POINTS), + named(name, NodeMetric::EVALUATED_GRID_POINTS), &self.evaluated_grid_points, ); count( set, partition, - path.metric_name(NodeMetric::OUTPUT_VALUES), + named(name, NodeMetric::OUTPUT_VALUES), &self.output_values, ); count( set, partition, - path.metric_name(NodeMetric::COMPLETED_SERIES), + named(name, NodeMetric::COMPLETED_SERIES), &self.completed_series, ); gauge( set, partition, - path.metric_name(NodeMetric::CURRENT_RESERVED_BYTES), + named(name, NodeMetric::CURRENT_RESERVED_BYTES), &self.current_reserved_bytes, ); // The peak is the one figure named per partition. The collector groups by name and sums @@ -271,25 +199,25 @@ impl NodeMetrics { gauge( set, partition, - path.partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, partition), + partitioned(name, NodeMetric::PEAK_RESERVED_BYTES, partition), &self.peak_reserved_bytes, ); gauge( set, partition, - path.metric_name(NodeMetric::GROUPS), + named(name, NodeMetric::GROUPS), &self.groups, ); gauge( set, partition, - path.metric_name(NodeMetric::STATE_ENTRIES), + named(name, NodeMetric::STATE_ENTRIES), &self.state_entries, ); MetricBuilder::new(set) .with_partition(partition) .build(MetricValue::Time { - name: path.metric_name(NodeMetric::SELF_COMPUTE_TIME_NS).into(), + name: named(name, NodeMetric::SELF_COMPUTE_TIME_NS).into(), time: self.self_compute_time.clone(), }); } @@ -341,269 +269,97 @@ impl NodeMetrics { } } -/// Registers every node of `root` in `set` and returns the handles in traversal order. -pub fn register_tree( - root: &NativeNode, - set: &ExecutionPlanMetricsSet, - partition: usize, -) -> Vec<(NodePath, Arc)> { - let mut out = Vec::new(); - walk(root, &mut |_, path| { - let metrics = Arc::new(NodeMetrics::default()); - metrics.register(path, set, partition); - out.push((path.clone(), metrics)); - }); - out -} - #[cfg(test)] mod tests { - use std::sync::Arc; - use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue}; - use datatypes::series::samples::SeriesSchema; - use datatypes::series::time::{EvalGrid, GridSpec}; - use super::{NodeMetric, NodePath, register_tree, walk}; - use crate::map::MathOp; - use crate::native::plan::{ - BoundInput, BoundMap, BoundWindow, MapOp, NativeExpr, NativeNode, NativeType, - }; - use crate::range_eval::{RangeFunction, RangeWindow}; + use super::{NodeMetric, NodeMetrics}; - fn grid() -> Arc { - Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 60_000, 4).unwrap()).unwrap()) + fn metrics_named(set: &ExecutionPlanMetricsSet) -> Vec { + set.clone_inner() + .iter() + .map(|metric| metric.value().name().to_string()) + .collect() } - fn schema() -> Arc { - Arc::new(SeriesSchema { - fields: Arc::new([]), - eval_grid: None, - }) - } - - fn input() -> NativeNode { - let output = NativeType::Samples(schema()); - NativeNode { - expr: NativeExpr::Input(BoundInput { - output: output.clone(), - ordering: datatypes::series::SeriesOrdering::KeyThenTime, - }), - output, + #[test] + fn fresh_exports_register_exactly_the_declared_metrics_without_growth() { + assert_eq!(NodeMetric::ALL.len(), 11); + let metrics = NodeMetrics::default(); + for _ in 0..2 { + let set = ExecutionPlanMetricsSet::new(); + metrics.register("root", &set, 0); + let names = metrics_named(&set); + assert_eq!(names.len(), NodeMetric::ALL.len()); + let mut names: Vec<_> = names + .iter() + .map(|name| name.rsplit('.').next().unwrap().to_string()) + .collect(); + names.sort(); + let mut declared: Vec<_> = NodeMetric::ALL + .iter() + .map(|name| name.to_string()) + .collect(); + declared.sort(); + assert_eq!(names, declared); } } - fn window(inner: NativeNode) -> NativeNode { - let grid = grid(); - NativeNode { - expr: NativeExpr::Window { - input: Box::new(inner), - op: BoundWindow { - function: RangeFunction::Rate, - window: RangeWindow { - range_ms: 300_000, - offset_ms: 0, - anchor_ms: None, - modifier: None, - }, - input_grid: None, - output_grid: Arc::clone(&grid), - }, - }, - output: NativeType::Samples(Arc::new(SeriesSchema { - fields: Arc::new([]), - eval_grid: Some(grid), - })), - } - } - - fn map(inner: NativeNode) -> NativeNode { - let grid = inner.output.grid().expect("a map needs a grid").clone(); - NativeNode { - expr: NativeExpr::Map { - input: Box::new(inner), - op: BoundMap { - ops: Arc::new([MapOp::Math(MathOp::Abs)]), - }, - }, - output: NativeType::Samples(Arc::new(SeriesSchema { - fields: Arc::new([]), - eval_grid: Some(grid), - })), - } - } - - /// A unary chain is addressed by `.input`, so a metric can be attributed to one node. #[test] - fn a_unary_chain_is_addressed_by_input_steps() { - let root = map(window(input())); - let mut paths = Vec::new(); - walk(&root, &mut |_, path| paths.push(path.to_string())); - assert_eq!(paths, vec!["root", "root.input", "root.input.input"]); - } - - /// A binary node addresses its two sides by name, so the two `rate` subtrees stay distinct. - #[test] - fn a_binary_node_addresses_both_sides() { - let binary = crate::native::plan::NativeNode { - expr: NativeExpr::Binary { - left: Box::new(window(input())), - right: Box::new(map(window(input()))), - matching: crate::vector_match::BinaryMatching { - op: crate::vector_match::BinaryOp::Add, - return_bool: false, - labels: crate::vector_match::MatchingLabels::Ignoring(Vec::new()), - cardinality: crate::vector_match::Cardinality::OneToOne, - fill: crate::vector_match::FillValues::default(), - }, - }, - output: { - let grid = grid(); - NativeType::Samples(Arc::new(SeriesSchema { - fields: Arc::new([]), - eval_grid: Some(grid), - })) - }, - }; - let mut paths = Vec::new(); - walk(&binary, &mut |_, path| paths.push(path.to_string())); - assert_eq!( - paths, - vec![ - "root", - "root.left", - "root.left.input", - "root.right", - "root.right.input", - "root.right.input.input", - ] - ); - } - - /// The name carries the position because the collector aggregates by name; a bare name would - /// merge two nodes of the same operation into one number. - #[test] - fn a_metric_name_carries_its_position() { - let path = NodePath::root().left().input(); - assert_eq!(path.as_str(), "root.left.input"); - assert_eq!( - path.metric_name(NodeMetric::SELF_COMPUTE_TIME_NS), - "native.root.left.input.self_compute_time_ns" - ); - assert_eq!( - path.partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 1), - "native.p1.root.left.input.peak_reserved_bytes" - ); - } - - /// The peak keeps the partition in its name, so the collector's aggregation by name cannot add - /// two partitions' peaks: 1024 and 2048 peak at different moments, and 3072 is neither peak. - #[test] - fn a_peak_is_not_merged_across_partitions() { - let root = map(window(input())); + fn exported_counters_have_recorded_values() { + let metrics = NodeMetrics::default(); + metrics.record_input(3, 17); + metrics.record_output(2, 11); let set = ExecutionPlanMetricsSet::new(); - let peaks = [(0, 1024), (1, 2048)]; - for (partition, peak) in peaks { - let registered = register_tree(&root, &set, partition); - // The root comes first in traversal order. - registered[0].1.record_reserved_bytes(peak); - } - - // The real collector path: `aggregate_by_name` groups by metric name and sums the - // values in each group. - let aggregated = set.clone_inner().aggregate_by_name(); - let peak = |partition: usize| -> Option { - let expected = NodePath::root() - .partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, partition); - aggregated.iter().find_map(|metric| match metric.value() { - MetricValue::Gauge { name, gauge } if name.as_ref() == expected.as_str() => { - Some(gauge.value()) - } + metrics.register("root", &set, 0); + let registered = set.clone_inner(); + let counter = |name: &str| { + registered.iter().find_map(|metric| match metric.value() { + MetricValue::Count { + name: metric_name, + count, + } if metric_name.as_ref() == name => Some(count.value()), _ => None, }) }; - // Two separate names, each holding its own partition's peak: had the name not carried the - // partition, aggregation would have merged them into one gauge summing to 3072. - assert_eq!(peak(0), Some(1024)); - assert_eq!(peak(1), Some(2048)); + assert_eq!(counter("native.root.input_batches"), Some(3)); + assert_eq!(counter("native.root.input_samples"), Some(17)); + assert_eq!(counter("native.root.output_batches"), Some(2)); + assert_eq!(counter("native.root.output_values"), Some(11)); } - /// Registration covers every node, so no node is executed without a way to observe it. - /// - /// The set is read back rather than named again from the tree, so this checks what - /// registration actually did. #[test] - fn registration_covers_every_node() { - let root = map(window(input())); - let set = datafusion::physical_plan::metrics::ExecutionPlanMetricsSet::new(); - let registered = register_tree(&root, &set, 2); - assert_eq!(registered.len(), 3); - - // Read the names back from the set registration wrote to, so the check is against what - // registration did rather than against the names built a second time here. - let set = set.clone_inner(); - let names: Vec<&str> = set.iter().map(|metric| metric.value().name()).collect(); - // One entry per node per declared metric, and every node's position among them: no node - // is executed without a way to observe it. - assert_eq!(names.len(), registered.len() * NodeMetric::ALL.len()); - for (path, _) in ®istered { - // The peak is named per partition, so a plain name stands for the node here. - let name = path.metric_name(NodeMetric::OUTPUT_VALUES); - assert!( - names.contains(&name.as_str()), - "{path} registered nothing under its own position" - ); - } + fn peak_exports_are_distinct_per_partition() { + let metrics = NodeMetrics::default(); + metrics.record_reserved_bytes(1024); + let first = ExecutionPlanMetricsSet::new(); + metrics.register("root", &first, 0); + metrics.record_reserved_bytes(2048); + let second = ExecutionPlanMetricsSet::new(); + metrics.register("root", &second, 1); + let peak = |set: &ExecutionPlanMetricsSet, expected: &str| { + set.clone_inner() + .iter() + .find_map(|metric| match metric.value() { + MetricValue::Gauge { name, gauge } if name.as_ref() == expected => { + Some(gauge.value()) + } + _ => None, + }) + }; + assert_eq!( + peak(&first, "native.p0.root.peak_reserved_bytes"), + Some(1024) + ); + assert_eq!( + peak(&second, "native.p1.root.peak_reserved_bytes"), + Some(2048) + ); } - /// Every metric a node reports is listed in one place, so registration cannot drift from the - /// documented set. - #[test] - fn every_metric_is_declared() { - assert_eq!(NodeMetric::ALL.len(), 11); - assert!(NodeMetric::ALL.contains(&NodeMetric::SELF_COMPUTE_TIME_NS)); - assert!(NodeMetric::ALL.contains(&NodeMetric::PEAK_RESERVED_BYTES)); - } - - /// Registration reports exactly the metrics `ALL` declares, in both directions. - /// - /// The names are read back from the set a node registered in, so the check is against what - /// registration actually did, not against `ALL` itself. Adding a metric to `ALL` without - /// registering it in `register`, or registering a metric that `ALL` does not list, fails this - /// test: it compares the two sides instead of trusting either. - #[test] - fn registered_metrics_are_exactly_the_declared_set() { - let root = map(window(input())); - let set = ExecutionPlanMetricsSet::new(); - register_tree(&root, &set, 0); - - // One node reports each metric once; the node paths and the partition prefix separate - // nodes, so the bare name is the last dot-separated segment. - let set = set.clone_inner(); - let mut registered: Vec<&str> = set - .iter() - .map(|metric| { - metric - .value() - .name() - .rsplit('.') - .next() - .expect("a metric name has a last segment") - }) - .collect(); - registered.sort_unstable(); - registered.dedup(); - - let mut declared: Vec<&str> = NodeMetric::ALL.to_vec(); - declared.sort_unstable(); - - assert_eq!(registered, declared); - } - - /// A peak is a high-water mark of one node, so it survives a later drop in reservations. #[test] fn a_peak_is_a_high_water_mark() { - let metrics = super::NodeMetrics::default(); + let metrics = NodeMetrics::default(); metrics.record_reserved_bytes(1024); metrics.record_reserved_bytes(64); assert_eq!(metrics.peak_reserved_bytes.value(), 1024); diff --git a/src/promql/tests/native_binary.rs b/src/promql/tests/native_binary.rs index a4e6f248d73..3362d47f4e4 100644 --- a/src/promql/tests/native_binary.rs +++ b/src/promql/tests/native_binary.rs @@ -23,7 +23,6 @@ use std::sync::Arc; use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; -use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datatypes::arrow::array::Float64Array; use datatypes::series::samples::{ FieldSpec, SampleRef, Samples, Series, SeriesFields, SeriesSchema, @@ -32,8 +31,7 @@ use datatypes::series::time::{EvalGrid, GridSpan, GridSpec, TimeAxis}; use datatypes::series::{Label, MetricName, SeriesOrdering}; use promql::evaluated::EvaluatedChunk; use promql::map::MathOp; -use promql::native::execute::{NativeExecutionState, NativeExecutor}; -use promql::native::metrics::NodePath; +use promql::native::execute::{InputId, NativeExecutionState, NativeExecutor}; use promql::native::plan::{ BoundInput, BoundMap, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, }; @@ -155,21 +153,17 @@ fn bound_tree(schema: &Arc) -> NativeTree { /// /// Nothing is handed out while the inputs are still arriving: a binary node needs both operands /// whole, so the chunks it is given are retained and the root answers on [`NativeExecutor::finish`]. -fn run(tree: &NativeTree, pushes: Vec<(&NodePath, EvaluatedChunk)>) -> Vec { +fn run(tree: &NativeTree, pushes: Vec<(InputId, EvaluatedChunk)>) -> Vec { let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 20)); - let metrics = ExecutionPlanMetricsSet::new(); - let state = NativeExecutionState::try_new(tree, Arc::clone(&pool), &metrics, 0) + let state = NativeExecutionState::try_new(tree, Arc::clone(&pool)) .expect("the runtime executes this tree"); let mut executor = NativeExecutor::new(state); - for (path, chunk) in pushes { + for (input_id, chunk) in pushes { executor - .push_evaluated_at(path, chunk) - .expect("the chunk is bound to the input it is pushed at"); + .push_evaluated(input_id, chunk) + .expect("the chunk is bound to the input slot it is pushed at"); let pulled = executor.next_chunk().expect("the pull succeeds"); - assert!( - pulled.is_none(), - "answered at {path} before its input ended" - ); + assert!(pulled.is_none(), "answered before its input ended"); } let chunks = executor.finish().expect("every series closed"); assert_eq!(executor.state().reserved_bytes(), 0, "nothing held"); @@ -214,11 +208,10 @@ fn public_nested_binary_map_is_addressed() { let grid = grid(); let schema = evaluated_schema(&grid); let tree = bound_tree(&schema); - // The root is the map; its input is the subtraction, whose right input is the division. - let minuend = NodePath::root().input(); - let a = minuend.left(); - let b = minuend.right().left(); - let c = minuend.right().right(); + // Inputs receive slots in left-to-right depth-first order: A, then B, then C. + let a = InputId::new(0); + let b = InputId::new(1); + let c = InputId::new(2); // What each input holds: A has no value at p3, B none at p1 and C none at p1 or p3. let a_rows: &[(usize, f64)] = &[(0, -2.0), (1, 100.0), (2, 10.0), (4, -3.0)]; @@ -229,9 +222,9 @@ fn public_nested_binary_map_is_addressed() { let whole = run( &tree, vec![ - (&b, chunk(&schema, "b", b_rows, true)), - (&c, chunk(&schema, "c", c_rows, true)), - (&a, chunk(&schema, "a", a_rows, true)), + (b, chunk(&schema, "b", b_rows, true)), + (c, chunk(&schema, "c", c_rows, true)), + (a, chunk(&schema, "a", a_rows, true)), ], ); assert_addressed_result(&whole, &grid); @@ -241,12 +234,12 @@ fn public_nested_binary_map_is_addressed() { let split = run( &tree, vec![ - (&b, chunk(&schema, "b", &b_rows[..1], false)), - (&c, chunk(&schema, "c", &c_rows[..1], false)), - (&a, chunk(&schema, "a", &a_rows[..2], false)), - (&c, chunk(&schema, "c", &c_rows[1..], true)), - (&a, chunk(&schema, "a", &a_rows[2..], true)), - (&b, chunk(&schema, "b", &b_rows[1..], true)), + (b, chunk(&schema, "b", &b_rows[..1], false)), + (c, chunk(&schema, "c", &c_rows[..1], false)), + (a, chunk(&schema, "a", &a_rows[..2], false)), + (c, chunk(&schema, "c", &c_rows[1..], true)), + (a, chunk(&schema, "a", &a_rows[2..], true)), + (b, chunk(&schema, "b", &b_rows[1..], true)), ], ); assert_addressed_result(&split, &grid); diff --git a/src/query/src/promql/native_compile/tests.rs b/src/query/src/promql/native_compile/tests.rs index 058cb4fc486..1d8d3ef9a45 100644 --- a/src/query/src/promql/native_compile/tests.rs +++ b/src/query/src/promql/native_compile/tests.rs @@ -17,7 +17,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use datafusion::error::DataFusionError; use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; -use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datatypes::arrow::array::Float64Array; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; use datatypes::series::SeriesOrdering; @@ -26,7 +25,7 @@ use datatypes::series::samples::{FieldSpec, SampleRef, Samples, SeriesFields, Se use datatypes::series::{Label, MetricName}; use promql::evaluated::EvaluatedChunk; use promql::map::MathOp; -use promql::native::execute::{NativeExecutionState, NativeExecutor}; +use promql::native::execute::{InputId, NativeExecutionState, NativeExecutor}; use promql::native::plan::{ AggregateKind, AggregateStage, BoundAggregate, BoundInstant, BoundMap, BoundWindow, Grouping, MapOp, NativeExpr, NativeNode, NativeTree, NativeType, @@ -2419,12 +2418,12 @@ fn execute_compiled(query: &str, chunks: Vec) -> Vec = Arc::new(GreedyMemoryPool::new(1 << 20)); let mut execution = NativeExecutor::new( - NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0) + NativeExecutionState::try_new(&tree, pool) .unwrap_or_else(|error| panic!("`{query}` was not executable: {error}")), ); for chunk in chunks { execution - .push_raw(chunk) + .push_raw(InputId::new(0), chunk) .unwrap_or_else(|error| panic!("`{query}` refused its input: {error}")); } let mut out = Vec::new(); @@ -3850,12 +3849,11 @@ fn aggregate_results(output: &[EvaluatedChunk]) -> Vec { /// decided at each absolute grid position. fn run_aggregate_tree(tree: &NativeTree, chunks: Vec) -> Vec { let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 26)); - let state = - NativeExecutionState::try_new(tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap(); + let state = NativeExecutionState::try_new(tree, pool).unwrap(); let mut execution = NativeExecutor::new(state); for chunk in chunks { execution - .push_raw(chunk) + .push_raw(InputId::new(0), chunk) .unwrap_or_else(|error| panic!("a source chunk was refused: {error}")); } let output = execution @@ -3952,11 +3950,10 @@ fn a_compiled_sum_aggregates_series_chunks_after_input_ends() { ), ]; let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 26)); - let state = - NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap(); + let state = NativeExecutionState::try_new(&tree, pool).unwrap(); let mut execution = NativeExecutor::new(state); for chunk in chunks { - execution.push_raw(chunk).unwrap(); + execution.push_raw(InputId::new(0), chunk).unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "`{query}` produced output before the whole input ended" @@ -4236,11 +4233,10 @@ fn a_compiled_sum_over_a_rate_aggregates_series_chunks_after_input_ends() { ), ]; let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 26)); - let state = - NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap(); + let state = NativeExecutionState::try_new(&tree, pool).unwrap(); let mut execution = NativeExecutor::new(state); for chunk in chunks { - execution.push_raw(chunk).unwrap(); + execution.push_raw(InputId::new(0), chunk).unwrap(); assert!( execution.next_chunk().unwrap().is_none(), "the rate aggregate produced output before the whole input ended"