mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 16:54:09 +00:00
refactor(promql): remove NodePath and own the runtime tree recursively
A node was identified by the path that reached it, so every execution and every metric export walked that path again, and a node's own state lived in tables keyed by it. This replaces that with one recursive runtime tree: each node owns its children, its operator state, its metrics and, once settled, nothing it no longer needs. An `InputId` is assigned only to the leaves that actually receive a feed, by a depth-first walk of the built tree, so a scalar leaf takes no id and no node is addressed by position. Pulling, settling, flushing after a subtree closes, EOF, errors and drop all recurse through the same tree. Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
@@ -61,7 +61,7 @@ use datafusion::execution::memory_pool::{
|
||||
};
|
||||
use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
|
||||
use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr};
|
||||
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
|
||||
use datafusion::physical_plan::metrics::MetricsSet;
|
||||
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
|
||||
use datafusion::physical_plan::{
|
||||
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements,
|
||||
@@ -76,8 +76,10 @@ use store_api::region_engine::{RawLabelSource, SendableRawSeriesStream};
|
||||
|
||||
use crate::evaluated::EvaluatedChunk;
|
||||
use crate::grid_series::{LabelSet, label_set};
|
||||
use crate::native::execute::InputId;
|
||||
use crate::native::execute::RuntimeTreeHandle;
|
||||
use crate::native::execute::{NativeExecutionState, NativeExecutor};
|
||||
use crate::native::metrics::{self, NodePath};
|
||||
use crate::native::metrics;
|
||||
use crate::native::plan::{NativeExpr, NativeTree};
|
||||
use crate::native::runs::{Run, absorb};
|
||||
use crate::source::materialize_raw_labels;
|
||||
@@ -404,9 +406,9 @@ impl PartialOrd for FusedSeries {
|
||||
/// can only agree or disagree with it, and the frame this node reads has to be decoded against the
|
||||
/// same schema, so both halves read it here instead of each carrying a copy.
|
||||
struct TreeInput {
|
||||
/// The position the tree's raw chunks enter at: a position is how the runtime addresses a node,
|
||||
/// so nothing on the node has to carry an identity that the tree itself does not.
|
||||
path: NodePath,
|
||||
/// Which of the tree's inputs the raw chunks enter at. A tree this node feeds has one leaf, so
|
||||
/// this is the first input; it is kept here so the exec does not have to count inputs again.
|
||||
input_id: InputId,
|
||||
/// The raw series schema the tree's input node was compiled against.
|
||||
schema: Arc<SeriesSchema>,
|
||||
}
|
||||
@@ -414,17 +416,17 @@ struct TreeInput {
|
||||
/// Reads what the tree's input node was compiled against, rejecting a tree this node cannot feed.
|
||||
///
|
||||
/// A node feeds one raw input, and its output is addressed on a grid; a tree that does not have
|
||||
/// that shape is one this layer cannot hand chunks to or read positions from.
|
||||
/// that shape is one this layer cannot hand chunks to. Because exactly one input is accepted, that
|
||||
/// input is the tree's only leaf and is therefore the first one: the id it is queued at is
|
||||
/// [`InputId::new(0)`], so nothing here has to count leaves beyond checking that it is alone.
|
||||
fn tree_input(tree: &NativeTree) -> DataFusionResult<TreeInput> {
|
||||
let mut sources = 0;
|
||||
let mut input_path = None;
|
||||
let mut schema = None;
|
||||
metrics::walk(tree.root(), &mut |node, path| {
|
||||
metrics::walk(tree.root(), &mut |node, _| {
|
||||
let NativeExpr::Input(input) = &node.expr else {
|
||||
return;
|
||||
};
|
||||
sources += 1;
|
||||
input_path.get_or_insert_with(|| path.clone());
|
||||
// Raw, not evaluated: this node feeds the tree from a scan, so a tree whose input was
|
||||
// compiled against an already-evaluated schema would read the wrong thing.
|
||||
schema = input
|
||||
@@ -438,7 +440,7 @@ fn tree_input(tree: &NativeTree) -> DataFusionResult<TreeInput> {
|
||||
"FusedSeries runs a tree with {sources} inputs, and it needs exactly one"
|
||||
)));
|
||||
}
|
||||
let (Some(path), Some(schema)) = (input_path, schema) else {
|
||||
let Some(schema) = schema else {
|
||||
return Err(DataFusionError::Plan(
|
||||
"FusedSeries runs a tree whose input is not raw series".to_string(),
|
||||
));
|
||||
@@ -453,7 +455,10 @@ fn tree_input(tree: &NativeTree) -> DataFusionResult<TreeInput> {
|
||||
"FusedSeries runs a tree whose output is not evaluated on a grid".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(TreeInput { path, schema })
|
||||
Ok(TreeInput {
|
||||
input_id: InputId::new(0),
|
||||
schema,
|
||||
})
|
||||
}
|
||||
|
||||
/// Refuses a tree whose logical series can be split across the partitions this node runs.
|
||||
@@ -749,8 +754,8 @@ pub struct FusedSeriesExec {
|
||||
label_source: RawLabelSource,
|
||||
/// The ordering the input frames guarantee.
|
||||
ordering: SeriesOrdering,
|
||||
/// The position of the tree's input node, where every raw chunk a partition reads enters.
|
||||
input_path: NodePath,
|
||||
/// Which of the tree's inputs every raw chunk a partition reads enters at.
|
||||
input_id: InputId,
|
||||
/// Where each output column comes from.
|
||||
classification: Arc<[OutputColumn]>,
|
||||
/// The output column holding the grid timestamps, kept so this node can be rebuilt over a new
|
||||
@@ -767,8 +772,8 @@ pub struct FusedSeriesExec {
|
||||
///
|
||||
/// Plan-level rather than per-partition: see [`LabelSets`].
|
||||
labels: SharedLabelSets,
|
||||
/// Per-node metrics, registered by each execution of the tree.
|
||||
metric: ExecutionPlanMetricsSet,
|
||||
/// Runtime roots from executions of the tree, retained for fresh metrics exports.
|
||||
runtimes: Mutex<Vec<(usize, RuntimeTreeHandle)>>,
|
||||
/// This node's partitioning, emission and boundedness.
|
||||
properties: Arc<PlanProperties>,
|
||||
}
|
||||
@@ -799,12 +804,7 @@ impl FusedSeriesExec {
|
||||
// Building a state is what proves the runtime executes this tree: a node kind it does not
|
||||
// run yet is refused here, so a plan that is built is a plan that starts. The throwaway
|
||||
// state is dropped at once, returning its (unbounded) reservations.
|
||||
let _ = NativeExecutionState::try_new(
|
||||
&tree,
|
||||
Arc::new(UnboundedMemoryPool::default()),
|
||||
&ExecutionPlanMetricsSet::new(),
|
||||
0,
|
||||
)?;
|
||||
let _ = NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()))?;
|
||||
let classification =
|
||||
classify_output(&output_schema, &tree_input.schema, &tags, &time_column)?;
|
||||
// Read here rather than below, because whether the split below can happen decides whether
|
||||
@@ -867,14 +867,14 @@ impl FusedSeriesExec {
|
||||
series_schema: tree_input.schema,
|
||||
label_source,
|
||||
ordering,
|
||||
input_path: tree_input.path,
|
||||
input_id: tree_input.input_id,
|
||||
classification: classification.into(),
|
||||
time_column,
|
||||
tags: tags.into(),
|
||||
input,
|
||||
output_schema,
|
||||
labels,
|
||||
metric: ExecutionPlanMetricsSet::new(),
|
||||
runtimes: Mutex::new(Vec::new()),
|
||||
properties,
|
||||
})
|
||||
}
|
||||
@@ -987,15 +987,18 @@ impl ExecutionPlan for FusedSeriesExec {
|
||||
let state = NativeExecutionState::try_new(
|
||||
&self.tree,
|
||||
Arc::clone(&context.runtime_env().memory_pool),
|
||||
&self.metric,
|
||||
partition,
|
||||
)?;
|
||||
let runtime = state.runtime_tree();
|
||||
self.runtimes
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.push((partition, runtime));
|
||||
// What the label sets hold is charged to the reservation attached above, so nothing is
|
||||
// reserved per stream: the entries outlive every stream and are released with the plan.
|
||||
Ok(Box::pin(FusedSeriesStream {
|
||||
input: Some(input),
|
||||
executor: Some(NativeExecutor::new(state)),
|
||||
input_path: self.input_path.clone(),
|
||||
input_id: self.input_id,
|
||||
classification: Arc::clone(&self.classification),
|
||||
tags: Arc::clone(&self.tags),
|
||||
output_schema: self.output_schema.clone(),
|
||||
@@ -1006,7 +1009,15 @@ impl ExecutionPlan for FusedSeriesExec {
|
||||
}
|
||||
|
||||
fn metrics(&self) -> Option<MetricsSet> {
|
||||
Some(self.metric.clone_inner())
|
||||
let runtimes = self.runtimes.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut metrics = MetricsSet::new();
|
||||
for (partition, runtime) in runtimes.iter() {
|
||||
let runtime = runtime.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for metric in runtime.export_metrics(*partition).iter() {
|
||||
metrics.push(Arc::clone(metric));
|
||||
}
|
||||
}
|
||||
Some(metrics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1368,7 +1379,7 @@ pub struct FusedSeriesStream {
|
||||
/// The execution of this node's tree, or `None` once it has been closed.
|
||||
executor: Option<NativeExecutor>,
|
||||
/// The position of the tree's input node, which is where every chunk read enters.
|
||||
input_path: NodePath,
|
||||
input_id: InputId,
|
||||
/// Where each output column comes from.
|
||||
classification: Arc<[OutputColumn]>,
|
||||
/// The tag column names, indexed by `OutputColumn::Tag`.
|
||||
@@ -1395,7 +1406,7 @@ impl FusedSeriesStream {
|
||||
};
|
||||
// The chunk enters at the tree's input node by position: a position is how the runtime
|
||||
// addresses a node, and a tree that gains a second input names them the same way.
|
||||
executor.push_raw_at(&self.input_path, chunk)?;
|
||||
executor.push_raw(self.input_id, chunk)?;
|
||||
let mut settled = Vec::new();
|
||||
while let Some(chunk) = executor.next_chunk()? {
|
||||
settled.push(chunk);
|
||||
|
||||
@@ -54,7 +54,7 @@ use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr};
|
||||
use datafusion::physical_optimizer::PhysicalOptimizerRule;
|
||||
use datafusion::physical_optimizer::ensure_requirements::EnsureRequirements;
|
||||
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
|
||||
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue, MetricsSet};
|
||||
use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};
|
||||
use datafusion::physical_plan::{
|
||||
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties,
|
||||
RecordBatchStream, SendableRecordBatchStream, collect, displayable,
|
||||
@@ -995,13 +995,8 @@ async fn a_runtime_error_after_rows_releases_the_input() {
|
||||
fn two_differently_named_series_collide_once_rate_drops_the_name() {
|
||||
let tree = rate_tree();
|
||||
let mut execution = NativeExecutor::new(
|
||||
NativeExecutionState::try_new(
|
||||
&tree,
|
||||
Arc::new(UnboundedMemoryPool::default()),
|
||||
&ExecutionPlanMetricsSet::new(),
|
||||
0,
|
||||
)
|
||||
.expect("the runtime executes a rate tree"),
|
||||
NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()))
|
||||
.expect("the runtime executes a rate tree"),
|
||||
);
|
||||
let samples = series_samples(1.0);
|
||||
execution
|
||||
@@ -1218,6 +1213,48 @@ async fn a_series_spanning_chunks_keeps_absolute_positions_and_closes_once() {
|
||||
.expect("the node reports its runtime's metrics");
|
||||
assert_eq!(1, count(&metrics, "native.root.input.completed_series"));
|
||||
assert_eq!(1, count(&metrics, "native.root.completed_series"));
|
||||
|
||||
// Exporting a snapshot repeatedly does not register another copy of the runtime metrics.
|
||||
let repeated = exec.metrics().expect("metrics can be exported repeatedly");
|
||||
assert_eq!(metrics.iter().count(), repeated.iter().count());
|
||||
assert_eq!(1, count(&repeated, "native.root.input.completed_series"));
|
||||
assert_eq!(1, count(&repeated, "native.root.completed_series"));
|
||||
}
|
||||
|
||||
/// Dropping a partially consumed execution releases its live reservation but preserves its peak.
|
||||
#[tokio::test]
|
||||
async fn cancelling_after_partial_input_clears_current_bytes_and_keeps_peak() {
|
||||
let schema = frame_schema();
|
||||
let partial = frame(&series_chunk("web-1", &series_samples(1.0)[..4], false));
|
||||
let exec = plan(
|
||||
rate_tree(),
|
||||
staged(schema.clone(), move || {
|
||||
Box::pin(StagedStream::new(schema.clone(), vec![partial.clone()]).pending_after(1))
|
||||
}),
|
||||
)
|
||||
.expect("the runtime executes a rate tree");
|
||||
let pool = Arc::new(GreedyMemoryPool::new(1 << 20));
|
||||
let mut stream = exec
|
||||
.execute(0, task_context(&pool))
|
||||
.expect("the node executes");
|
||||
|
||||
// One unclosed series chunk is accepted; the source then parks before another arrives.
|
||||
assert!(futures::poll!(stream.next()).is_pending());
|
||||
drop(stream);
|
||||
|
||||
let metrics = exec.metrics().expect("the runtime metrics are exported");
|
||||
let current = metrics
|
||||
.sum_by_name("native.root.current_reserved_bytes")
|
||||
.expect("the current-bytes gauge is exported");
|
||||
let peak = metrics
|
||||
.sum_by_name("native.p0.root.peak_reserved_bytes")
|
||||
.expect("the peak-bytes gauge is exported");
|
||||
assert!(matches!(current, MetricValue::Gauge { gauge, .. } if gauge.value() == 0));
|
||||
assert!(matches!(peak, MetricValue::Gauge { gauge, .. } if gauge.value() > 0));
|
||||
assert!(
|
||||
pool.reserved() > 0,
|
||||
"plan-level label-set charge is independent"
|
||||
);
|
||||
}
|
||||
|
||||
/// A tree the runtime does not execute is refused when the execution plan is built.
|
||||
|
||||
@@ -38,8 +38,7 @@ use crate::aggregate::{AggregateOp, FloatAggregator, Grouping};
|
||||
use crate::evaluated::EvaluatedChunk;
|
||||
use crate::grid_series::series_bytes;
|
||||
use crate::map::validate_float_series;
|
||||
use crate::native::execute::{NodeBuffer, evaluated_bytes};
|
||||
use crate::native::metrics::NodePath;
|
||||
use crate::native::execute::{evaluated_bytes, NodeBuffer};
|
||||
use crate::native::plan::{
|
||||
AggregateKind, AggregateStage, BoundAggregate, Grouping as BoundGrouping,
|
||||
};
|
||||
@@ -152,7 +151,6 @@ impl AggregateRuntime {
|
||||
/// what makes a group's answer its whole group's answer once the input ends.
|
||||
pub(crate) fn accumulate(
|
||||
&mut self,
|
||||
path: &NodePath,
|
||||
chunk: EvaluatedChunk,
|
||||
buffer: &mut NodeBuffer,
|
||||
pending: usize,
|
||||
@@ -161,8 +159,12 @@ impl AggregateRuntime {
|
||||
// another schema; the states are indexed by absolute position on this node's grid, so a
|
||||
// chunk of another field count or another grid would be folded into the wrong places.
|
||||
validate_float_series(&self.schema, &chunk.series)
|
||||
.map_err(|error| folded_chunk(path, &self.describe(), error))?;
|
||||
let accumulator = self.accumulator.as_mut().ok_or_else(|| no_fold(path))?;
|
||||
.map_err(|error| folded_chunk(&self.describe(), error))?;
|
||||
let aggregate = self.describe();
|
||||
let accumulator = self
|
||||
.accumulator
|
||||
.as_mut()
|
||||
.ok_or_else(|| no_fold(&aggregate))?;
|
||||
// Keep the chunk covered while the fold decides whether it needs a new group or block.
|
||||
accumulator.update_budgeted(&chunk.series, |fold_bytes| {
|
||||
buffer.resize(pending.saturating_add(fold_bytes))
|
||||
@@ -183,7 +185,6 @@ impl AggregateRuntime {
|
||||
/// a fold whose states are gone.
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
path: &NodePath,
|
||||
buffer: &mut NodeBuffer,
|
||||
queue_bytes: usize,
|
||||
) -> Result<Option<Vec<Series>>> {
|
||||
@@ -191,13 +192,13 @@ impl AggregateRuntime {
|
||||
return Ok(None);
|
||||
}
|
||||
if self.accumulator.is_none() {
|
||||
return Err(no_fold(path));
|
||||
return Err(no_fold(&self.describe()));
|
||||
}
|
||||
self.computed = true;
|
||||
let output = self
|
||||
.accumulator
|
||||
.take()
|
||||
.ok_or_else(|| no_fold(path))?
|
||||
.ok_or_else(|| no_fold(&self.describe()))?
|
||||
.finish_budgeted(|total| buffer.resize(total.saturating_add(queue_bytes)))?;
|
||||
let output_bytes = output.iter().fold(0usize, |bytes, series| {
|
||||
bytes.saturating_add(series_bytes(series))
|
||||
@@ -225,19 +226,19 @@ impl AggregateRuntime {
|
||||
/// kernel saying it does not compute this shape at all -- a native histogram in a float aggregate,
|
||||
/// for instance -- and a caller that falls back to another planner on `NotImplemented` still can.
|
||||
/// Wrapping that one in an `Execution` would deny it.
|
||||
fn folded_chunk(path: &NodePath, aggregate: &str, error: DataFusionError) -> DataFusionError {
|
||||
fn folded_chunk(aggregate: &str, error: DataFusionError) -> DataFusionError {
|
||||
match error {
|
||||
DataFusionError::NotImplemented(_) => error,
|
||||
error => DataFusionError::Execution(format!(
|
||||
"the `{aggregate}` aggregate at {path} folds chunks bound to its own schema, and this \
|
||||
"the `{aggregate}` aggregate folds chunks bound to its own schema, and this \
|
||||
one is not: {error}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The error for a fold that has already been finished.
|
||||
fn no_fold(path: &NodePath) -> DataFusionError {
|
||||
fn no_fold(aggregate: &str) -> DataFusionError {
|
||||
DataFusionError::Execution(format!(
|
||||
"the aggregate at {path} already finished its fold, so it takes no more input"
|
||||
"the `{aggregate}` aggregate already finished its fold, so it takes no more input"
|
||||
))
|
||||
}
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
|
||||
//! One line per node of a [`NativeTree`].
|
||||
//!
|
||||
//! The position a node is shown at is derived by [`metrics::walk`], the same traversal that names
|
||||
//! the node's metrics, so a node cannot be shown at one place and measured at another. The
|
||||
//! display is an observation of a compiled tree: it runs nothing, and every line reads as the node
|
||||
//! it is, making no claim about whether that node could run.
|
||||
//! The tree is displayed by recursively visiting its bound expressions. The display is an
|
||||
//! observation of a compiled tree: it runs nothing, and every line reads as the node it is,
|
||||
//! making no claim about whether that node could run.
|
||||
|
||||
use std::fmt::{self, Write};
|
||||
use std::sync::Arc;
|
||||
@@ -27,7 +26,6 @@ use datatypes::series::{MetricName, SeriesOrdering};
|
||||
|
||||
use crate::histogram_eval::HistogramFunction;
|
||||
use crate::map::MathOp;
|
||||
use crate::native::metrics::{self, NodePath};
|
||||
use crate::native::plan::{
|
||||
AggregateKind, AggregateStage, BoundVectorOp, BoundWindow, Grouping, MapOp, NativeExpr,
|
||||
NativeNode, NativeTree, NativeType,
|
||||
@@ -39,17 +37,6 @@ use crate::vector_match::{BinaryOp, Cardinality, FillValues, MatchingLabels};
|
||||
/// The header line of every tree.
|
||||
const HEADER: &str = "NativeTree";
|
||||
|
||||
/// How deep `path` is: the root is at depth zero, its input at one.
|
||||
fn depth(path: &NodePath) -> usize {
|
||||
path.as_str().matches('.').count()
|
||||
}
|
||||
|
||||
/// The last segment of `path`, which is the name of the node in its parent.
|
||||
fn name(path: &NodePath) -> &str {
|
||||
// `NodePath` only ever appends `.segment`, so the last segment is never empty.
|
||||
path.as_str().rsplit('.').next().unwrap_or(path.as_str())
|
||||
}
|
||||
|
||||
/// A grid as `<start>/<step>x<len>`.
|
||||
fn grid_name(grid: &EvalGrid) -> String {
|
||||
let spec = grid.spec();
|
||||
@@ -346,29 +333,39 @@ fn kind(expr: &NativeExpr) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// The line a node is shown as, without its indentation.
|
||||
fn line(node: &NativeNode, path: &NodePath) -> String {
|
||||
format!("{} {} {}", name(path), kind(&node.expr), describe(node))
|
||||
/// One recursively displayed node and its children.
|
||||
fn display_node(
|
||||
f: &mut fmt::Formatter<'_>,
|
||||
node: &NativeNode,
|
||||
depth: usize,
|
||||
edge: &str,
|
||||
) -> fmt::Result {
|
||||
f.write_char('\n')?;
|
||||
// The root sits one level in from the header, and each child one level below its consumer.
|
||||
for _ in 0..depth + 1 {
|
||||
f.write_str(" ")?;
|
||||
}
|
||||
write!(f, "{edge} {} {}", kind(&node.expr), describe(node))?;
|
||||
match &node.expr {
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) => {}
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => display_node(f, input, depth + 1, "input")?,
|
||||
NativeExpr::Binary { left, right, .. } => {
|
||||
display_node(f, left, depth + 1, "left")?;
|
||||
display_node(f, right, depth + 1, "right")?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl fmt::Display for NativeTree {
|
||||
/// One line per node, outermost first, indented two spaces per depth below the header.
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(HEADER)?;
|
||||
let mut lines = Vec::new();
|
||||
metrics::walk(self.root(), &mut |node, path| {
|
||||
lines.push((depth(path), line(node, path)));
|
||||
});
|
||||
for (depth, line) in lines {
|
||||
f.write_char('\n')?;
|
||||
// The root sits one level in from the header, and each input one level below its
|
||||
// consumer, so the shape of the tree is the shape of the output.
|
||||
for _ in 0..depth + 1 {
|
||||
f.write_str(" ")?;
|
||||
}
|
||||
f.write_str(&line)?;
|
||||
}
|
||||
Ok(())
|
||||
display_node(f, self.root(), 0, "root")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+824
-1274
File diff suppressed because it is too large
Load Diff
+1016
-948
File diff suppressed because it is too large
Load Diff
+102
-346
@@ -12,105 +12,19 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Where a node sits in the final execution tree, and the metrics named after that position.
|
||||
//! Per-node metric handles for native PromQL execution and their temporary DataFusion export.
|
||||
//!
|
||||
//! A node is identified by its structural position — `root`, `.input`, `.left`, `.right` — rather
|
||||
//! than by an id it carries. The position is not stored on the node: it is derived while walking
|
||||
//! the tree. [`walk`] is the one traversal rule, and [`register_tree`] is its caller.
|
||||
//!
|
||||
//! The position stays in the metric name because the collector aggregates by name first: two
|
||||
//! `rate` nodes sharing a bare name would be merged into one number. Positions are only an
|
||||
//! observation detail of one plan — they are not stable across queries, plan rewrites or versions,
|
||||
//! so nothing may depend on them beyond correlating a metric with a tree it was taken from.
|
||||
|
||||
use std::sync::Arc;
|
||||
//! Node metrics retain no names or tree topology. Runtime callers may export their handles into a
|
||||
//! fresh temporary DataFusion metrics set using names collected for that export.
|
||||
|
||||
use datafusion::physical_plan::metrics::MetricValue;
|
||||
use datafusion::physical_plan::metrics::{
|
||||
Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, Time,
|
||||
};
|
||||
|
||||
use crate::native::plan::{NativeExpr, NativeNode};
|
||||
|
||||
/// The reserved first segment of every native metric name.
|
||||
const NATIVE_PREFIX: &str = "native";
|
||||
|
||||
/// A node's position in the final tree.
|
||||
///
|
||||
/// Rendering allocates a short string once, when the run-time state is created; the hot path holds
|
||||
/// the metric handles instead of looking anything up by name.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct NodePath(String);
|
||||
|
||||
impl NodePath {
|
||||
/// The position of the root node.
|
||||
pub fn root() -> Self {
|
||||
Self("root".to_string())
|
||||
}
|
||||
|
||||
/// The position of a unary node's input.
|
||||
pub fn input(&self) -> Self {
|
||||
Self(format!("{}.input", self.0))
|
||||
}
|
||||
|
||||
/// The position of a binary node's left input.
|
||||
pub fn left(&self) -> Self {
|
||||
Self(format!("{}.left", self.0))
|
||||
}
|
||||
|
||||
/// The position of a binary node's right input.
|
||||
pub fn right(&self) -> Self {
|
||||
Self(format!("{}.right", self.0))
|
||||
}
|
||||
|
||||
/// The path without the metric namespace.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// The full metric name for `metric` at this position.
|
||||
pub fn metric_name(&self, metric: &str) -> String {
|
||||
format!("{NATIVE_PREFIX}.{}.{}", self.0, metric)
|
||||
}
|
||||
|
||||
/// The full metric name for `metric` at this position in `partition`.
|
||||
///
|
||||
/// A figure that must not be merged across partitions needs the partition in its name: the
|
||||
/// collector aggregates by name first, and a sum of peaks is not a peak.
|
||||
pub fn partitioned_metric_name(&self, metric: &str, partition: usize) -> String {
|
||||
format!("{NATIVE_PREFIX}.p{partition}.{}.{metric}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for NodePath {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Visits every node of `root` with its position, outermost first.
|
||||
///
|
||||
/// This is the one traversal rule: [`register_tree`] is its caller, so a node's metric is always
|
||||
/// named after the position this walk gives it.
|
||||
pub fn walk(root: &NativeNode, visit: &mut dyn FnMut(&NativeNode, &NodePath)) {
|
||||
fn go(node: &NativeNode, path: &NodePath, visit: &mut dyn FnMut(&NativeNode, &NodePath)) {
|
||||
visit(node, path);
|
||||
match &node.expr {
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) => {}
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => go(input, &path.input(), visit),
|
||||
NativeExpr::Binary { left, right, .. } => {
|
||||
go(left, &path.left(), visit);
|
||||
go(right, &path.right(), visit);
|
||||
}
|
||||
}
|
||||
}
|
||||
go(root, &NodePath::root(), visit);
|
||||
}
|
||||
|
||||
/// The metric names one node reports.
|
||||
///
|
||||
/// [`Self::ALL`] is the list a reader should treat as complete: [`NodeMetrics::register`] names each
|
||||
@@ -196,13 +110,27 @@ impl Default for NodeMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
/// The full metric name for `metric` reported by the node called `name`.
|
||||
fn named(name: &str, metric: &str) -> String {
|
||||
format!("{NATIVE_PREFIX}.{name}.{metric}")
|
||||
}
|
||||
|
||||
/// The full metric name for `metric` in `partition` of the node called `name`.
|
||||
///
|
||||
/// A figure that must not be merged across partitions needs the partition in its name: the
|
||||
/// collector aggregates by name first, and a sum of peaks is not a peak.
|
||||
fn partitioned(name: &str, metric: &str, partition: usize) -> String {
|
||||
format!("{NATIVE_PREFIX}.p{partition}.{name}.{metric}")
|
||||
}
|
||||
|
||||
impl NodeMetrics {
|
||||
/// Registers this node's metrics under `path` in `set`.
|
||||
/// Exports this node's metrics under its temporary `name` into `set`.
|
||||
///
|
||||
/// `partition` is part of the registration, so two partitions of one plan report separately.
|
||||
/// It is also part of the peak's name, because a peak is the one figure a collector must not
|
||||
/// add up across partitions.
|
||||
pub fn register(&self, path: &NodePath, set: &ExecutionPlanMetricsSet, partition: usize) {
|
||||
/// Callers must invoke this only when collecting a runtime execution into a fresh temporary
|
||||
/// metrics set. Names belong to that export and are not retained by this node. `partition` is
|
||||
/// part of the DataFusion registration, and the peak name includes it because peaks must not be
|
||||
/// summed across partitions.
|
||||
pub fn register(&self, name: &str, set: &ExecutionPlanMetricsSet, partition: usize) {
|
||||
fn count(set: &ExecutionPlanMetricsSet, partition: usize, name: String, count: &Count) {
|
||||
MetricBuilder::new(set)
|
||||
.with_partition(partition)
|
||||
@@ -223,43 +151,43 @@ impl NodeMetrics {
|
||||
count(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::INPUT_BATCHES),
|
||||
named(name, NodeMetric::INPUT_BATCHES),
|
||||
&self.input_batches,
|
||||
);
|
||||
count(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::OUTPUT_BATCHES),
|
||||
named(name, NodeMetric::OUTPUT_BATCHES),
|
||||
&self.output_batches,
|
||||
);
|
||||
count(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::INPUT_SAMPLES),
|
||||
named(name, NodeMetric::INPUT_SAMPLES),
|
||||
&self.input_samples,
|
||||
);
|
||||
count(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::EVALUATED_GRID_POINTS),
|
||||
named(name, NodeMetric::EVALUATED_GRID_POINTS),
|
||||
&self.evaluated_grid_points,
|
||||
);
|
||||
count(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::OUTPUT_VALUES),
|
||||
named(name, NodeMetric::OUTPUT_VALUES),
|
||||
&self.output_values,
|
||||
);
|
||||
count(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::COMPLETED_SERIES),
|
||||
named(name, NodeMetric::COMPLETED_SERIES),
|
||||
&self.completed_series,
|
||||
);
|
||||
gauge(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::CURRENT_RESERVED_BYTES),
|
||||
named(name, NodeMetric::CURRENT_RESERVED_BYTES),
|
||||
&self.current_reserved_bytes,
|
||||
);
|
||||
// The peak is the one figure named per partition. The collector groups by name and sums
|
||||
@@ -271,25 +199,25 @@ impl NodeMetrics {
|
||||
gauge(
|
||||
set,
|
||||
partition,
|
||||
path.partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, partition),
|
||||
partitioned(name, NodeMetric::PEAK_RESERVED_BYTES, partition),
|
||||
&self.peak_reserved_bytes,
|
||||
);
|
||||
gauge(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::GROUPS),
|
||||
named(name, NodeMetric::GROUPS),
|
||||
&self.groups,
|
||||
);
|
||||
gauge(
|
||||
set,
|
||||
partition,
|
||||
path.metric_name(NodeMetric::STATE_ENTRIES),
|
||||
named(name, NodeMetric::STATE_ENTRIES),
|
||||
&self.state_entries,
|
||||
);
|
||||
MetricBuilder::new(set)
|
||||
.with_partition(partition)
|
||||
.build(MetricValue::Time {
|
||||
name: path.metric_name(NodeMetric::SELF_COMPUTE_TIME_NS).into(),
|
||||
name: named(name, NodeMetric::SELF_COMPUTE_TIME_NS).into(),
|
||||
time: self.self_compute_time.clone(),
|
||||
});
|
||||
}
|
||||
@@ -341,269 +269,97 @@ impl NodeMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers every node of `root` in `set` and returns the handles in traversal order.
|
||||
pub fn register_tree(
|
||||
root: &NativeNode,
|
||||
set: &ExecutionPlanMetricsSet,
|
||||
partition: usize,
|
||||
) -> Vec<(NodePath, Arc<NodeMetrics>)> {
|
||||
let mut out = Vec::new();
|
||||
walk(root, &mut |_, path| {
|
||||
let metrics = Arc::new(NodeMetrics::default());
|
||||
metrics.register(path, set, partition);
|
||||
out.push((path.clone(), metrics));
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue};
|
||||
use datatypes::series::samples::SeriesSchema;
|
||||
use datatypes::series::time::{EvalGrid, GridSpec};
|
||||
|
||||
use super::{NodeMetric, NodePath, register_tree, walk};
|
||||
use crate::map::MathOp;
|
||||
use crate::native::plan::{
|
||||
BoundInput, BoundMap, BoundWindow, MapOp, NativeExpr, NativeNode, NativeType,
|
||||
};
|
||||
use crate::range_eval::{RangeFunction, RangeWindow};
|
||||
use super::{NodeMetric, NodeMetrics};
|
||||
|
||||
fn grid() -> Arc<EvalGrid> {
|
||||
Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 60_000, 4).unwrap()).unwrap())
|
||||
fn metrics_named(set: &ExecutionPlanMetricsSet) -> Vec<String> {
|
||||
set.clone_inner()
|
||||
.iter()
|
||||
.map(|metric| metric.value().name().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn schema() -> Arc<SeriesSchema> {
|
||||
Arc::new(SeriesSchema {
|
||||
fields: Arc::new([]),
|
||||
eval_grid: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn input() -> NativeNode {
|
||||
let output = NativeType::Samples(schema());
|
||||
NativeNode {
|
||||
expr: NativeExpr::Input(BoundInput {
|
||||
output: output.clone(),
|
||||
ordering: datatypes::series::SeriesOrdering::KeyThenTime,
|
||||
}),
|
||||
output,
|
||||
#[test]
|
||||
fn fresh_exports_register_exactly_the_declared_metrics_without_growth() {
|
||||
assert_eq!(NodeMetric::ALL.len(), 11);
|
||||
let metrics = NodeMetrics::default();
|
||||
for _ in 0..2 {
|
||||
let set = ExecutionPlanMetricsSet::new();
|
||||
metrics.register("root", &set, 0);
|
||||
let names = metrics_named(&set);
|
||||
assert_eq!(names.len(), NodeMetric::ALL.len());
|
||||
let mut names: Vec<_> = names
|
||||
.iter()
|
||||
.map(|name| name.rsplit('.').next().unwrap().to_string())
|
||||
.collect();
|
||||
names.sort();
|
||||
let mut declared: Vec<_> = NodeMetric::ALL
|
||||
.iter()
|
||||
.map(|name| name.to_string())
|
||||
.collect();
|
||||
declared.sort();
|
||||
assert_eq!(names, declared);
|
||||
}
|
||||
}
|
||||
|
||||
fn window(inner: NativeNode) -> NativeNode {
|
||||
let grid = grid();
|
||||
NativeNode {
|
||||
expr: NativeExpr::Window {
|
||||
input: Box::new(inner),
|
||||
op: BoundWindow {
|
||||
function: RangeFunction::Rate,
|
||||
window: RangeWindow {
|
||||
range_ms: 300_000,
|
||||
offset_ms: 0,
|
||||
anchor_ms: None,
|
||||
modifier: None,
|
||||
},
|
||||
input_grid: None,
|
||||
output_grid: Arc::clone(&grid),
|
||||
},
|
||||
},
|
||||
output: NativeType::Samples(Arc::new(SeriesSchema {
|
||||
fields: Arc::new([]),
|
||||
eval_grid: Some(grid),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
fn map(inner: NativeNode) -> NativeNode {
|
||||
let grid = inner.output.grid().expect("a map needs a grid").clone();
|
||||
NativeNode {
|
||||
expr: NativeExpr::Map {
|
||||
input: Box::new(inner),
|
||||
op: BoundMap {
|
||||
ops: Arc::new([MapOp::Math(MathOp::Abs)]),
|
||||
},
|
||||
},
|
||||
output: NativeType::Samples(Arc::new(SeriesSchema {
|
||||
fields: Arc::new([]),
|
||||
eval_grid: Some(grid),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// A unary chain is addressed by `.input`, so a metric can be attributed to one node.
|
||||
#[test]
|
||||
fn a_unary_chain_is_addressed_by_input_steps() {
|
||||
let root = map(window(input()));
|
||||
let mut paths = Vec::new();
|
||||
walk(&root, &mut |_, path| paths.push(path.to_string()));
|
||||
assert_eq!(paths, vec!["root", "root.input", "root.input.input"]);
|
||||
}
|
||||
|
||||
/// A binary node addresses its two sides by name, so the two `rate` subtrees stay distinct.
|
||||
#[test]
|
||||
fn a_binary_node_addresses_both_sides() {
|
||||
let binary = crate::native::plan::NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(window(input())),
|
||||
right: Box::new(map(window(input()))),
|
||||
matching: crate::vector_match::BinaryMatching {
|
||||
op: crate::vector_match::BinaryOp::Add,
|
||||
return_bool: false,
|
||||
labels: crate::vector_match::MatchingLabels::Ignoring(Vec::new()),
|
||||
cardinality: crate::vector_match::Cardinality::OneToOne,
|
||||
fill: crate::vector_match::FillValues::default(),
|
||||
},
|
||||
},
|
||||
output: {
|
||||
let grid = grid();
|
||||
NativeType::Samples(Arc::new(SeriesSchema {
|
||||
fields: Arc::new([]),
|
||||
eval_grid: Some(grid),
|
||||
}))
|
||||
},
|
||||
};
|
||||
let mut paths = Vec::new();
|
||||
walk(&binary, &mut |_, path| paths.push(path.to_string()));
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec![
|
||||
"root",
|
||||
"root.left",
|
||||
"root.left.input",
|
||||
"root.right",
|
||||
"root.right.input",
|
||||
"root.right.input.input",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// The name carries the position because the collector aggregates by name; a bare name would
|
||||
/// merge two nodes of the same operation into one number.
|
||||
#[test]
|
||||
fn a_metric_name_carries_its_position() {
|
||||
let path = NodePath::root().left().input();
|
||||
assert_eq!(path.as_str(), "root.left.input");
|
||||
assert_eq!(
|
||||
path.metric_name(NodeMetric::SELF_COMPUTE_TIME_NS),
|
||||
"native.root.left.input.self_compute_time_ns"
|
||||
);
|
||||
assert_eq!(
|
||||
path.partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, 1),
|
||||
"native.p1.root.left.input.peak_reserved_bytes"
|
||||
);
|
||||
}
|
||||
|
||||
/// The peak keeps the partition in its name, so the collector's aggregation by name cannot add
|
||||
/// two partitions' peaks: 1024 and 2048 peak at different moments, and 3072 is neither peak.
|
||||
#[test]
|
||||
fn a_peak_is_not_merged_across_partitions() {
|
||||
let root = map(window(input()));
|
||||
fn exported_counters_have_recorded_values() {
|
||||
let metrics = NodeMetrics::default();
|
||||
metrics.record_input(3, 17);
|
||||
metrics.record_output(2, 11);
|
||||
let set = ExecutionPlanMetricsSet::new();
|
||||
let peaks = [(0, 1024), (1, 2048)];
|
||||
for (partition, peak) in peaks {
|
||||
let registered = register_tree(&root, &set, partition);
|
||||
// The root comes first in traversal order.
|
||||
registered[0].1.record_reserved_bytes(peak);
|
||||
}
|
||||
|
||||
// The real collector path: `aggregate_by_name` groups by metric name and sums the
|
||||
// values in each group.
|
||||
let aggregated = set.clone_inner().aggregate_by_name();
|
||||
let peak = |partition: usize| -> Option<usize> {
|
||||
let expected = NodePath::root()
|
||||
.partitioned_metric_name(NodeMetric::PEAK_RESERVED_BYTES, partition);
|
||||
aggregated.iter().find_map(|metric| match metric.value() {
|
||||
MetricValue::Gauge { name, gauge } if name.as_ref() == expected.as_str() => {
|
||||
Some(gauge.value())
|
||||
}
|
||||
metrics.register("root", &set, 0);
|
||||
let registered = set.clone_inner();
|
||||
let counter = |name: &str| {
|
||||
registered.iter().find_map(|metric| match metric.value() {
|
||||
MetricValue::Count {
|
||||
name: metric_name,
|
||||
count,
|
||||
} if metric_name.as_ref() == name => Some(count.value()),
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
// Two separate names, each holding its own partition's peak: had the name not carried the
|
||||
// partition, aggregation would have merged them into one gauge summing to 3072.
|
||||
assert_eq!(peak(0), Some(1024));
|
||||
assert_eq!(peak(1), Some(2048));
|
||||
assert_eq!(counter("native.root.input_batches"), Some(3));
|
||||
assert_eq!(counter("native.root.input_samples"), Some(17));
|
||||
assert_eq!(counter("native.root.output_batches"), Some(2));
|
||||
assert_eq!(counter("native.root.output_values"), Some(11));
|
||||
}
|
||||
|
||||
/// Registration covers every node, so no node is executed without a way to observe it.
|
||||
///
|
||||
/// The set is read back rather than named again from the tree, so this checks what
|
||||
/// registration actually did.
|
||||
#[test]
|
||||
fn registration_covers_every_node() {
|
||||
let root = map(window(input()));
|
||||
let set = datafusion::physical_plan::metrics::ExecutionPlanMetricsSet::new();
|
||||
let registered = register_tree(&root, &set, 2);
|
||||
assert_eq!(registered.len(), 3);
|
||||
|
||||
// Read the names back from the set registration wrote to, so the check is against what
|
||||
// registration did rather than against the names built a second time here.
|
||||
let set = set.clone_inner();
|
||||
let names: Vec<&str> = set.iter().map(|metric| metric.value().name()).collect();
|
||||
// One entry per node per declared metric, and every node's position among them: no node
|
||||
// is executed without a way to observe it.
|
||||
assert_eq!(names.len(), registered.len() * NodeMetric::ALL.len());
|
||||
for (path, _) in ®istered {
|
||||
// The peak is named per partition, so a plain name stands for the node here.
|
||||
let name = path.metric_name(NodeMetric::OUTPUT_VALUES);
|
||||
assert!(
|
||||
names.contains(&name.as_str()),
|
||||
"{path} registered nothing under its own position"
|
||||
);
|
||||
}
|
||||
fn peak_exports_are_distinct_per_partition() {
|
||||
let metrics = NodeMetrics::default();
|
||||
metrics.record_reserved_bytes(1024);
|
||||
let first = ExecutionPlanMetricsSet::new();
|
||||
metrics.register("root", &first, 0);
|
||||
metrics.record_reserved_bytes(2048);
|
||||
let second = ExecutionPlanMetricsSet::new();
|
||||
metrics.register("root", &second, 1);
|
||||
let peak = |set: &ExecutionPlanMetricsSet, expected: &str| {
|
||||
set.clone_inner()
|
||||
.iter()
|
||||
.find_map(|metric| match metric.value() {
|
||||
MetricValue::Gauge { name, gauge } if name.as_ref() == expected => {
|
||||
Some(gauge.value())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
assert_eq!(
|
||||
peak(&first, "native.p0.root.peak_reserved_bytes"),
|
||||
Some(1024)
|
||||
);
|
||||
assert_eq!(
|
||||
peak(&second, "native.p1.root.peak_reserved_bytes"),
|
||||
Some(2048)
|
||||
);
|
||||
}
|
||||
|
||||
/// Every metric a node reports is listed in one place, so registration cannot drift from the
|
||||
/// documented set.
|
||||
#[test]
|
||||
fn every_metric_is_declared() {
|
||||
assert_eq!(NodeMetric::ALL.len(), 11);
|
||||
assert!(NodeMetric::ALL.contains(&NodeMetric::SELF_COMPUTE_TIME_NS));
|
||||
assert!(NodeMetric::ALL.contains(&NodeMetric::PEAK_RESERVED_BYTES));
|
||||
}
|
||||
|
||||
/// Registration reports exactly the metrics `ALL` declares, in both directions.
|
||||
///
|
||||
/// The names are read back from the set a node registered in, so the check is against what
|
||||
/// registration actually did, not against `ALL` itself. Adding a metric to `ALL` without
|
||||
/// registering it in `register`, or registering a metric that `ALL` does not list, fails this
|
||||
/// test: it compares the two sides instead of trusting either.
|
||||
#[test]
|
||||
fn registered_metrics_are_exactly_the_declared_set() {
|
||||
let root = map(window(input()));
|
||||
let set = ExecutionPlanMetricsSet::new();
|
||||
register_tree(&root, &set, 0);
|
||||
|
||||
// One node reports each metric once; the node paths and the partition prefix separate
|
||||
// nodes, so the bare name is the last dot-separated segment.
|
||||
let set = set.clone_inner();
|
||||
let mut registered: Vec<&str> = set
|
||||
.iter()
|
||||
.map(|metric| {
|
||||
metric
|
||||
.value()
|
||||
.name()
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.expect("a metric name has a last segment")
|
||||
})
|
||||
.collect();
|
||||
registered.sort_unstable();
|
||||
registered.dedup();
|
||||
|
||||
let mut declared: Vec<&str> = NodeMetric::ALL.to_vec();
|
||||
declared.sort_unstable();
|
||||
|
||||
assert_eq!(registered, declared);
|
||||
}
|
||||
|
||||
/// A peak is a high-water mark of one node, so it survives a later drop in reservations.
|
||||
#[test]
|
||||
fn a_peak_is_a_high_water_mark() {
|
||||
let metrics = super::NodeMetrics::default();
|
||||
let metrics = NodeMetrics::default();
|
||||
metrics.record_reserved_bytes(1024);
|
||||
metrics.record_reserved_bytes(64);
|
||||
assert_eq!(metrics.peak_reserved_bytes.value(), 1024);
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool};
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datatypes::arrow::array::Float64Array;
|
||||
use datatypes::series::samples::{
|
||||
FieldSpec, SampleRef, Samples, Series, SeriesFields, SeriesSchema,
|
||||
@@ -32,8 +31,7 @@ use datatypes::series::time::{EvalGrid, GridSpan, GridSpec, TimeAxis};
|
||||
use datatypes::series::{Label, MetricName, SeriesOrdering};
|
||||
use promql::evaluated::EvaluatedChunk;
|
||||
use promql::map::MathOp;
|
||||
use promql::native::execute::{NativeExecutionState, NativeExecutor};
|
||||
use promql::native::metrics::NodePath;
|
||||
use promql::native::execute::{InputId, NativeExecutionState, NativeExecutor};
|
||||
use promql::native::plan::{
|
||||
BoundInput, BoundMap, MapOp, NativeExpr, NativeNode, NativeTree, NativeType,
|
||||
};
|
||||
@@ -155,21 +153,17 @@ fn bound_tree(schema: &Arc<SeriesSchema>) -> NativeTree {
|
||||
///
|
||||
/// Nothing is handed out while the inputs are still arriving: a binary node needs both operands
|
||||
/// whole, so the chunks it is given are retained and the root answers on [`NativeExecutor::finish`].
|
||||
fn run(tree: &NativeTree, pushes: Vec<(&NodePath, EvaluatedChunk)>) -> Vec<EvaluatedChunk> {
|
||||
fn run(tree: &NativeTree, pushes: Vec<(InputId, EvaluatedChunk)>) -> Vec<EvaluatedChunk> {
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 20));
|
||||
let metrics = ExecutionPlanMetricsSet::new();
|
||||
let state = NativeExecutionState::try_new(tree, Arc::clone(&pool), &metrics, 0)
|
||||
let state = NativeExecutionState::try_new(tree, Arc::clone(&pool))
|
||||
.expect("the runtime executes this tree");
|
||||
let mut executor = NativeExecutor::new(state);
|
||||
for (path, chunk) in pushes {
|
||||
for (input_id, chunk) in pushes {
|
||||
executor
|
||||
.push_evaluated_at(path, chunk)
|
||||
.expect("the chunk is bound to the input it is pushed at");
|
||||
.push_evaluated(input_id, chunk)
|
||||
.expect("the chunk is bound to the input slot it is pushed at");
|
||||
let pulled = executor.next_chunk().expect("the pull succeeds");
|
||||
assert!(
|
||||
pulled.is_none(),
|
||||
"answered at {path} before its input ended"
|
||||
);
|
||||
assert!(pulled.is_none(), "answered before its input ended");
|
||||
}
|
||||
let chunks = executor.finish().expect("every series closed");
|
||||
assert_eq!(executor.state().reserved_bytes(), 0, "nothing held");
|
||||
@@ -214,11 +208,10 @@ fn public_nested_binary_map_is_addressed() {
|
||||
let grid = grid();
|
||||
let schema = evaluated_schema(&grid);
|
||||
let tree = bound_tree(&schema);
|
||||
// The root is the map; its input is the subtraction, whose right input is the division.
|
||||
let minuend = NodePath::root().input();
|
||||
let a = minuend.left();
|
||||
let b = minuend.right().left();
|
||||
let c = minuend.right().right();
|
||||
// Inputs receive slots in left-to-right depth-first order: A, then B, then C.
|
||||
let a = InputId::new(0);
|
||||
let b = InputId::new(1);
|
||||
let c = InputId::new(2);
|
||||
|
||||
// What each input holds: A has no value at p3, B none at p1 and C none at p1 or p3.
|
||||
let a_rows: &[(usize, f64)] = &[(0, -2.0), (1, 100.0), (2, 10.0), (4, -3.0)];
|
||||
@@ -229,9 +222,9 @@ fn public_nested_binary_map_is_addressed() {
|
||||
let whole = run(
|
||||
&tree,
|
||||
vec![
|
||||
(&b, chunk(&schema, "b", b_rows, true)),
|
||||
(&c, chunk(&schema, "c", c_rows, true)),
|
||||
(&a, chunk(&schema, "a", a_rows, true)),
|
||||
(b, chunk(&schema, "b", b_rows, true)),
|
||||
(c, chunk(&schema, "c", c_rows, true)),
|
||||
(a, chunk(&schema, "a", a_rows, true)),
|
||||
],
|
||||
);
|
||||
assert_addressed_result(&whole, &grid);
|
||||
@@ -241,12 +234,12 @@ fn public_nested_binary_map_is_addressed() {
|
||||
let split = run(
|
||||
&tree,
|
||||
vec![
|
||||
(&b, chunk(&schema, "b", &b_rows[..1], false)),
|
||||
(&c, chunk(&schema, "c", &c_rows[..1], false)),
|
||||
(&a, chunk(&schema, "a", &a_rows[..2], false)),
|
||||
(&c, chunk(&schema, "c", &c_rows[1..], true)),
|
||||
(&a, chunk(&schema, "a", &a_rows[2..], true)),
|
||||
(&b, chunk(&schema, "b", &b_rows[1..], true)),
|
||||
(b, chunk(&schema, "b", &b_rows[..1], false)),
|
||||
(c, chunk(&schema, "c", &c_rows[..1], false)),
|
||||
(a, chunk(&schema, "a", &a_rows[..2], false)),
|
||||
(c, chunk(&schema, "c", &c_rows[1..], true)),
|
||||
(a, chunk(&schema, "a", &a_rows[2..], true)),
|
||||
(b, chunk(&schema, "b", &b_rows[1..], true)),
|
||||
],
|
||||
);
|
||||
assert_addressed_result(&split, &grid);
|
||||
|
||||
@@ -17,7 +17,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use datafusion::error::DataFusionError;
|
||||
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool};
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datatypes::arrow::array::Float64Array;
|
||||
use datatypes::arrow::buffer::{Buffer, ScalarBuffer};
|
||||
use datatypes::series::SeriesOrdering;
|
||||
@@ -26,7 +25,7 @@ use datatypes::series::samples::{FieldSpec, SampleRef, Samples, SeriesFields, Se
|
||||
use datatypes::series::{Label, MetricName};
|
||||
use promql::evaluated::EvaluatedChunk;
|
||||
use promql::map::MathOp;
|
||||
use promql::native::execute::{NativeExecutionState, NativeExecutor};
|
||||
use promql::native::execute::{InputId, NativeExecutionState, NativeExecutor};
|
||||
use promql::native::plan::{
|
||||
AggregateKind, AggregateStage, BoundAggregate, BoundInstant, BoundMap, BoundWindow, Grouping,
|
||||
MapOp, NativeExpr, NativeNode, NativeTree, NativeType,
|
||||
@@ -2419,12 +2418,12 @@ fn execute_compiled(query: &str, chunks: Vec<RawSeriesChunk>) -> Vec<EvaluatedCh
|
||||
.unwrap_or_else(|error| panic!("`{query}` was refused: {error}"));
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 20));
|
||||
let mut execution = NativeExecutor::new(
|
||||
NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0)
|
||||
NativeExecutionState::try_new(&tree, pool)
|
||||
.unwrap_or_else(|error| panic!("`{query}` was not executable: {error}")),
|
||||
);
|
||||
for chunk in chunks {
|
||||
execution
|
||||
.push_raw(chunk)
|
||||
.push_raw(InputId::new(0), chunk)
|
||||
.unwrap_or_else(|error| panic!("`{query}` refused its input: {error}"));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
@@ -3850,12 +3849,11 @@ fn aggregate_results(output: &[EvaluatedChunk]) -> Vec<AggSeries> {
|
||||
/// decided at each absolute grid position.
|
||||
fn run_aggregate_tree(tree: &NativeTree, chunks: Vec<RawSeriesChunk>) -> Vec<AggSeries> {
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 26));
|
||||
let state =
|
||||
NativeExecutionState::try_new(tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap();
|
||||
let state = NativeExecutionState::try_new(tree, pool).unwrap();
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for chunk in chunks {
|
||||
execution
|
||||
.push_raw(chunk)
|
||||
.push_raw(InputId::new(0), chunk)
|
||||
.unwrap_or_else(|error| panic!("a source chunk was refused: {error}"));
|
||||
}
|
||||
let output = execution
|
||||
@@ -3952,11 +3950,10 @@ fn a_compiled_sum_aggregates_series_chunks_after_input_ends() {
|
||||
),
|
||||
];
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 26));
|
||||
let state =
|
||||
NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap();
|
||||
let state = NativeExecutionState::try_new(&tree, pool).unwrap();
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for chunk in chunks {
|
||||
execution.push_raw(chunk).unwrap();
|
||||
execution.push_raw(InputId::new(0), chunk).unwrap();
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"`{query}` produced output before the whole input ended"
|
||||
@@ -4236,11 +4233,10 @@ fn a_compiled_sum_over_a_rate_aggregates_series_chunks_after_input_ends() {
|
||||
),
|
||||
];
|
||||
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 26));
|
||||
let state =
|
||||
NativeExecutionState::try_new(&tree, pool, &ExecutionPlanMetricsSet::new(), 0).unwrap();
|
||||
let state = NativeExecutionState::try_new(&tree, pool).unwrap();
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for chunk in chunks {
|
||||
execution.push_raw(chunk).unwrap();
|
||||
execution.push_raw(InputId::new(0), chunk).unwrap();
|
||||
assert!(
|
||||
execution.next_chunk().unwrap().is_none(),
|
||||
"the rate aggregate produced output before the whole input ended"
|
||||
|
||||
Reference in New Issue
Block a user