mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 16:24:07 +00:00
feat(promql): evaluate dynamic scalar parameters natively (#10093)
Evaluate scalar expressions per step in native topk/bottomk and histogram functions, including scalar(vector) parameters that read independent selectors. Reuse the existing kernels, preserve whole-vector placement and selector binding order, account for retained parameter values, and append expression wire tags without renumbering existing variants. Expression-column naming is intentionally outside this change. Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
@@ -1239,6 +1239,21 @@ pub enum InputLocality {
|
||||
SinglePartition,
|
||||
}
|
||||
|
||||
/// Whether `node` answers its values without reading any source.
|
||||
///
|
||||
/// The set is the planner's `data_free` one: a constant, the evaluation time, a binary operation
|
||||
/// between those and `vector()` of one. It is what [`same_source`] needs on the scalar side of a
|
||||
/// broadcast: a scalar is not the same thing as a source-free value, because `scalar(a)` is a scalar
|
||||
/// that has read a whole vector to become one.
|
||||
fn reads_no_source(node: &NativeNode) -> bool {
|
||||
match &node.expr {
|
||||
NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => true,
|
||||
NativeExpr::Binary { left, right, .. } => reads_no_source(left) && reads_no_source(right),
|
||||
NativeExpr::VectorOf { input } => reads_no_source(input),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The one source every leaf of `node` reads, when the tree keeps the matching key of that source.
|
||||
fn same_source(node: &NativeNode) -> Option<usize> {
|
||||
match &node.expr {
|
||||
@@ -1277,7 +1292,7 @@ fn same_source(node: &NativeNode) -> Option<usize> {
|
||||
left,
|
||||
right,
|
||||
matching,
|
||||
} if matches!(left.output, NativeType::Scalar(_)) => {
|
||||
} if matches!(left.output, NativeType::Scalar(_)) && reads_no_source(left) => {
|
||||
let _ = matching;
|
||||
same_source(right)
|
||||
}
|
||||
@@ -1285,7 +1300,7 @@ fn same_source(node: &NativeNode) -> Option<usize> {
|
||||
left,
|
||||
right,
|
||||
matching,
|
||||
} if matches!(right.output, NativeType::Scalar(_)) => {
|
||||
} if matches!(right.output, NativeType::Scalar(_)) && reads_no_source(right) => {
|
||||
let _ = matching;
|
||||
same_source(left)
|
||||
}
|
||||
@@ -1310,7 +1325,12 @@ fn same_source(node: &NativeNode) -> Option<usize> {
|
||||
| NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::Map { .. }
|
||||
| NativeExpr::ParameterizedMap { .. } => None,
|
||||
| NativeExpr::ParameterizedMap { .. }
|
||||
// `scalar()`, `topk` and a histogram function read the whole vector at a point, so they
|
||||
// answer one value (or a whole ranking) rather than one output series per input series.
|
||||
| NativeExpr::ScalarOf { .. }
|
||||
| NativeExpr::ParameterizedTopK { .. }
|
||||
| NativeExpr::ParameterizedHistogram { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1441,10 +1461,17 @@ fn needs_vector_matching(node: &NativeNode) -> bool {
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => needs_vector_matching(input),
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::ScalarOf { input } => needs_vector_matching(input),
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => needs_vector_matching(input) || parameters.iter().any(needs_vector_matching),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => needs_vector_matching(input) || needs_vector_matching(parameter),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => needs_vector_matching(input) || parameters.iter().any(needs_vector_matching),
|
||||
NativeExpr::Set { left, right, .. } => {
|
||||
needs_vector_matching(left) || needs_vector_matching(right)
|
||||
}
|
||||
@@ -1456,7 +1483,11 @@ fn needs_vector_matching(node: &NativeNode) -> bool {
|
||||
/// The vector/vector match this tree performs, looking through the pointwise nodes wrapped on top.
|
||||
///
|
||||
/// A match pairs series of two whole vectors, so where it sits — including under a scalar
|
||||
/// broadcast or a pointwise wrapper — is the place that needs matching-key co-location.
|
||||
/// broadcast or a pointwise wrapper — is the place that needs matching-key co-location. The scalar
|
||||
/// side has to be a value that reads no source for that to hold: a scalar-output side that read a
|
||||
/// whole vector (`scalar(c)`) only exists once every series of it is gathered, so a match under it is
|
||||
/// not the unit of locality and the tree needs one partition instead. Answering `None` here leaves
|
||||
/// that decision to [`reads_across_series`].
|
||||
fn matching_site(node: &NativeNode) -> Option<(&NativeNode, &NativeNode, &MatchingLabels)> {
|
||||
match &node.expr {
|
||||
NativeExpr::Binary {
|
||||
@@ -1464,7 +1495,9 @@ fn matching_site(node: &NativeNode) -> Option<(&NativeNode, &NativeNode, &Matchi
|
||||
right,
|
||||
matching,
|
||||
} => {
|
||||
let scalar = |side: &NativeNode| matches!(side.output, NativeType::Scalar(_));
|
||||
let scalar = |side: &NativeNode| {
|
||||
matches!(side.output, NativeType::Scalar(_)) && reads_no_source(side)
|
||||
};
|
||||
if scalar(left) {
|
||||
matching_site(right)
|
||||
} else if scalar(right) {
|
||||
@@ -1546,15 +1579,25 @@ fn tree_inputs_for_locality(root: &NativeNode) -> Vec<&crate::native::plan::Boun
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => visit(input, inputs),
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::ScalarOf { input } => visit(input, inputs),
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
visit(input, inputs);
|
||||
for parameter in parameters {
|
||||
visit(parameter, inputs);
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
visit(input, inputs);
|
||||
visit(parameter, inputs);
|
||||
}
|
||||
NativeExpr::Binary { left, right, .. } | NativeExpr::Set { left, right, .. } => {
|
||||
visit(left, inputs);
|
||||
visit(right, inputs);
|
||||
@@ -1586,6 +1629,12 @@ fn state_routing(root: &NativeNode) -> Option<(StateGroup, SeriesRouting)> {
|
||||
input,
|
||||
histogram || matches!(op, crate::native::plan::BoundVectorOp::Histogram(_)),
|
||||
),
|
||||
// A parameterized histogram is a histogram suffix too: its function folds the bounds of
|
||||
// one histogram, so every bound has to meet where it runs.
|
||||
NativeExpr::ParameterizedHistogram { input, op, .. } => find(
|
||||
input,
|
||||
histogram || matches!(op, crate::native::plan::BoundVectorOp::Histogram(_)),
|
||||
),
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
@@ -1593,6 +1642,8 @@ fn state_routing(root: &NativeNode) -> Option<(StateGroup, SeriesRouting)> {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ParameterizedMap { input, .. }
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::ParameterizedTopK { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. } => find(input, histogram),
|
||||
NativeExpr::Binary { left, right, .. } | NativeExpr::Set { left, right, .. } => {
|
||||
find(left, histogram).or_else(|| find(right, histogram))
|
||||
@@ -1680,7 +1731,22 @@ fn tree_inputs(tree: &NativeTree) -> Vec<&crate::native::plan::BoundInput> {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::VectorOf { input } => collect(input, inputs),
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => collect(input, inputs),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
collect(input, inputs);
|
||||
for parameter in parameters {
|
||||
collect(parameter, inputs);
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
collect(input, inputs);
|
||||
collect(parameter, inputs);
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
@@ -1728,15 +1794,25 @@ fn partitioned_window_sources(node: &NativeNode) -> DataFusionResult<Vec<usize>>
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::VectorOf { input } => visit(input, sources)?,
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => visit(input, sources)?,
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
visit(input, sources)?;
|
||||
for parameter in parameters {
|
||||
visit(parameter, sources)?;
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
visit(input, sources)?;
|
||||
visit(parameter, sources)?;
|
||||
}
|
||||
NativeExpr::Binary { left, right, .. }
|
||||
| NativeExpr::Set { left, right, .. }
|
||||
| NativeExpr::Info {
|
||||
@@ -2071,6 +2147,7 @@ fn native_tree_has_vector_binary(node: &NativeNode) -> bool {
|
||||
|| native_tree_has_vector_binary(right)
|
||||
}
|
||||
NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Window { input, .. }
|
||||
@@ -2078,8 +2155,14 @@ fn native_tree_has_vector_binary(node: &NativeNode) -> bool {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => native_tree_has_vector_binary(input),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => native_tree_has_vector_binary(input) || native_tree_has_vector_binary(parameter),
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
native_tree_has_vector_binary(input)
|
||||
|| parameters.iter().any(native_tree_has_vector_binary)
|
||||
|
||||
@@ -94,7 +94,7 @@ use crate::native::execute::{InputId, NativeExecutionState, NativeExecutor};
|
||||
use crate::native::plan::{
|
||||
AggregateKind, AggregateStage, AggregateStateSchema, BoundAggregate, BoundInput, BoundInstant,
|
||||
BoundMap, BoundScalar, BoundVectorOp, BoundWindow, Grouping, InstantValue, MapOp, NativeExpr,
|
||||
NativeNode, NativeTree, NativeType,
|
||||
NativeNode, NativeTree, NativeType, ParameterizedOp,
|
||||
};
|
||||
use crate::range_eval::{RangeFunction, RangeWindow};
|
||||
use crate::vector_match::{BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels};
|
||||
@@ -405,6 +405,67 @@ fn evaluated_of(schema: &SeriesSchema, grid: &Arc<EvalGrid>) -> NativeType {
|
||||
}))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_evidence_detects_binaries_in_dynamic_nodes() {
|
||||
use crate::histogram_eval::HistogramFunction;
|
||||
use crate::native::plan::BoundVectorOp;
|
||||
|
||||
let grid = grid();
|
||||
let binary = binary_tree(raw_schema(), raw_schema()).root().clone();
|
||||
let scalar_of = |input| NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(input),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let no_binary = instant_of(0, &grid);
|
||||
let literal = scalar_tree(Arc::clone(&grid), 1.0).root().clone();
|
||||
let topk = |input: NativeNode, parameter| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
},
|
||||
};
|
||||
let histogram = |input: NativeNode, parameters| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Fraction {
|
||||
lower: 0.0,
|
||||
upper: 0.0,
|
||||
}),
|
||||
parameters,
|
||||
},
|
||||
};
|
||||
let has_binary = super::native_tree_has_vector_binary;
|
||||
assert!(has_binary(&scalar_of(binary.clone())));
|
||||
assert!(!has_binary(&scalar_of(no_binary.clone())));
|
||||
assert!(has_binary(&topk(binary.clone(), literal.clone())));
|
||||
assert!(has_binary(&topk(
|
||||
no_binary.clone(),
|
||||
scalar_of(binary.clone())
|
||||
)));
|
||||
assert!(!has_binary(&topk(no_binary.clone(), literal.clone())));
|
||||
assert!(has_binary(&histogram(
|
||||
binary.clone(),
|
||||
vec![literal.clone(), literal.clone()]
|
||||
)));
|
||||
assert!(has_binary(&histogram(
|
||||
no_binary.clone(),
|
||||
vec![literal.clone(), scalar_of(binary)]
|
||||
)));
|
||||
assert!(!has_binary(&histogram(
|
||||
no_binary,
|
||||
vec![literal.clone(), literal]
|
||||
)));
|
||||
}
|
||||
|
||||
/// `tree` with the orderings its input nodes declare replaced, in the order the nodes are reached.
|
||||
///
|
||||
/// The fixtures here declare `Contiguous`, which is what the plans below them present, so a test
|
||||
@@ -428,7 +489,8 @@ fn declaring_orderings(tree: &NativeTree, orderings: &[SeriesOrdering]) -> Nativ
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::VectorOf { input } => {
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => {
|
||||
**input = rewrite(input, orderings, seen);
|
||||
}
|
||||
NativeExpr::Binary { left, right, .. }
|
||||
@@ -441,6 +503,20 @@ fn declaring_orderings(tree: &NativeTree, orderings: &[SeriesOrdering]) -> Nativ
|
||||
**left = rewrite(left, orderings, seen);
|
||||
**right = rewrite(right, orderings, seen);
|
||||
}
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
**input = rewrite(input, orderings, seen);
|
||||
for parameter in parameters {
|
||||
*parameter = rewrite(parameter, orderings, seen);
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
**input = rewrite(input, orderings, seen);
|
||||
**parameter = rewrite(parameter, orderings, seen);
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
@@ -763,6 +839,303 @@ fn math_map_matching_key_preservation_controls_locality() {
|
||||
));
|
||||
}
|
||||
|
||||
/// The inner ranking's per-point parameter children finish before the outer ranking settles.
|
||||
/// Dropping those children must not prevent the outer `topk` from returning its suffix rows.
|
||||
#[tokio::test]
|
||||
async fn constant_topk_over_parameterized_topk_keeps_suffix_rows() {
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 60_000, 3).unwrap()).unwrap());
|
||||
let raw = NativeType::Samples(raw_schema());
|
||||
let output = evaluated_of(&raw_schema(), &grid);
|
||||
let window = NativeNode {
|
||||
expr: NativeExpr::Window {
|
||||
input: Box::new(NativeNode {
|
||||
expr: NativeExpr::Input(BoundInput {
|
||||
source_index: 0,
|
||||
output: raw.clone(),
|
||||
ordering: SeriesOrdering::Contiguous,
|
||||
}),
|
||||
output: raw,
|
||||
}),
|
||||
op: BoundWindow {
|
||||
function: RangeFunction::LastOverTime,
|
||||
window: RangeWindow {
|
||||
range_ms: 60_000,
|
||||
offset_ms: 0,
|
||||
anchor_ms: None,
|
||||
modifier: None,
|
||||
},
|
||||
input_grid: None,
|
||||
output_grid: Arc::clone(&grid),
|
||||
},
|
||||
},
|
||||
output: output.clone(),
|
||||
};
|
||||
let ceil = NativeNode {
|
||||
expr: NativeExpr::Map {
|
||||
input: Box::new(window),
|
||||
op: BoundMap {
|
||||
ops: Arc::from([MapOp::Math(crate::map::MathOp::Ceil)]),
|
||||
},
|
||||
},
|
||||
output: output.clone(),
|
||||
};
|
||||
let sum = NativeNode {
|
||||
expr: NativeExpr::Aggregate {
|
||||
input: Box::new(ceil),
|
||||
op: BoundAggregate {
|
||||
kind: AggregateKind::Sum,
|
||||
group: Grouping::By(Arc::from([Arc::from(HOST)])),
|
||||
stage: AggregateStage::Complete,
|
||||
},
|
||||
},
|
||||
output: output.clone(),
|
||||
};
|
||||
let scalar = |value| NativeNode {
|
||||
expr: NativeExpr::Scalar(BoundScalar { value }),
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let binary = |left, right, op| NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(left),
|
||||
right: Box::new(right),
|
||||
matching: BinaryMatching {
|
||||
op,
|
||||
return_bool: false,
|
||||
labels: MatchingLabels::Ignoring(Vec::new()),
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill: FillValues::default(),
|
||||
},
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let time = NativeNode {
|
||||
expr: NativeExpr::EvalTime { date: None },
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let parameter = binary(
|
||||
binary(time, scalar(1.0 / 60.0), BinaryOp::Mul),
|
||||
scalar(1.0),
|
||||
BinaryOp::Add,
|
||||
);
|
||||
let rank = |k| BoundVectorOp::TopK {
|
||||
k,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
};
|
||||
let inner = NativeNode {
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(sum),
|
||||
op: rank(0.0),
|
||||
parameter: Box::new(parameter),
|
||||
},
|
||||
output: output.clone(),
|
||||
};
|
||||
let tree = NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::Vector {
|
||||
input: Box::new(inner),
|
||||
op: rank(1.0),
|
||||
},
|
||||
output,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// `h0` through `h7` sum to 0 through 7. Splitting `h7` across the two physical
|
||||
// partitions also proves that the gathered execution reads both contributions: partition 0
|
||||
// alone would rank `h6=6` over its `h7=3` fragment.
|
||||
let mut partitions = vec![Vec::new(), Vec::new()];
|
||||
for host in 0..8 {
|
||||
let host_name = format!("h{host}");
|
||||
let parts: Vec<(usize, &str, f64)> = if host == 7 {
|
||||
vec![(0, "a", 3.0), (1, "b", 4.0)]
|
||||
} else {
|
||||
vec![(host % 2, "a", host as f64)]
|
||||
};
|
||||
for (partition, job, value) in parts {
|
||||
let key = format!("{host_name}-{job}");
|
||||
let samples = [
|
||||
(-60_000, value),
|
||||
(0, value),
|
||||
(60_000, value),
|
||||
(120_000, value),
|
||||
];
|
||||
partitions[partition].push(frame(&labelled_chunk(
|
||||
&key,
|
||||
&[(HOST, &host_name), (JOB, job)],
|
||||
&samples,
|
||||
true,
|
||||
)));
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
super::input_locality(tree.root()),
|
||||
super::InputLocality::SinglePartition
|
||||
));
|
||||
let exec = plan(tree, memory_exec_partitioned(partitions)).unwrap();
|
||||
let mut actual = rows(
|
||||
&collect(exec, SessionContext::default().task_ctx())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
actual.sort_by_key(|row| row.0);
|
||||
assert_eq!(
|
||||
actual,
|
||||
vec![
|
||||
(0, Some(7.0), Some("h7".to_string()), None),
|
||||
(60_000, Some(7.0), Some("h7".to_string()), None),
|
||||
(120_000, Some(7.0), Some("h7".to_string()), None),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// A node that reads the whole vector at a point answers one value (or a whole ranking) rather than
|
||||
/// one output series per input series, so it cannot keep a source's matching key: it needs every
|
||||
/// series of that source in one place.
|
||||
///
|
||||
/// Each of these shapes takes a per-point `time()` or a literal parameter, which is exactly the case
|
||||
/// a parameter-shaped `same_source` arm was tempted to pass through. Answering `PartitionLocal` here
|
||||
/// does not only cost parallelism: a `topk` or a histogram function run once per partition answers
|
||||
/// the union of the per-partition rankings.
|
||||
#[test]
|
||||
fn whole_vector_parameter_nodes_need_every_series_in_one_partition() {
|
||||
let grid = grid();
|
||||
let eval_time = || NativeNode {
|
||||
expr: NativeExpr::EvalTime { date: None },
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let single_partition = |tree: NativeTree| {
|
||||
assert!(
|
||||
matches!(
|
||||
super::input_locality(tree.root()),
|
||||
super::InputLocality::SinglePartition
|
||||
),
|
||||
"a whole-vector node must read its source in one partition: {tree}"
|
||||
);
|
||||
};
|
||||
|
||||
// `scalar(m)`: which series carries the one value at a point decides the answer.
|
||||
let instant = instant_of(0, &grid);
|
||||
single_partition(
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(instant),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
})
|
||||
.expect("scalar() over a raw selection is a legal tree"),
|
||||
);
|
||||
|
||||
// A per-point `k` decides which of the whole vector's series survive.
|
||||
let instant = instant_of(0, &grid);
|
||||
let output = instant.output.clone();
|
||||
single_partition(
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(instant),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(eval_time()),
|
||||
},
|
||||
output,
|
||||
})
|
||||
.expect("topk with a per-point k over a raw selection is a legal tree"),
|
||||
);
|
||||
|
||||
// The histogram function folds the bucket series of the whole vector into one.
|
||||
let rate = rate_tree();
|
||||
let output = rate.root().output.clone();
|
||||
single_partition(
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(rate.root().clone()),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.5)),
|
||||
parameters: vec![eval_time()],
|
||||
},
|
||||
output,
|
||||
})
|
||||
.expect("a histogram function over a raw rate window is a legal tree"),
|
||||
);
|
||||
|
||||
// `b * scalar(a)`: the scalar side is not source-free, so its broadcast has to wait until the
|
||||
// whole of its own vector is gathered.
|
||||
let vector_side = instant_of(0, &grid);
|
||||
let output = vector_side.output.clone();
|
||||
let scalar_side = NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(instant_of(1, &grid)),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
single_partition(
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(vector_side),
|
||||
right: Box::new(scalar_side),
|
||||
matching: BinaryMatching {
|
||||
op: BinaryOp::Mul,
|
||||
return_bool: false,
|
||||
labels: MatchingLabels::Ignoring(Vec::new()),
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill: FillValues::default(),
|
||||
},
|
||||
},
|
||||
output,
|
||||
})
|
||||
.expect("a scalar broadcast over a raw selection is a legal tree"),
|
||||
);
|
||||
|
||||
// `clamp_min(b, scalar(a))`: a parameterized map is only as local as its parameter.
|
||||
let instant = instant_of(0, &grid);
|
||||
let output = instant.output.clone();
|
||||
single_partition(
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::ParameterizedMap {
|
||||
input: Box::new(instant),
|
||||
op: ParameterizedOp::ClampMin,
|
||||
parameters: vec![NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(instant_of(1, &grid)),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
}],
|
||||
},
|
||||
output,
|
||||
})
|
||||
.expect("a parameterized map over a scalar() parameter is a legal tree"),
|
||||
);
|
||||
|
||||
// `(a + on(job) b) * scalar(c)`: the match sits under a broadcast whose scalar side reads a
|
||||
// third source, so the match's key is not the unit of locality. Routing by it would answer
|
||||
// `scalar(c)` once per key class, and a statement whose global answer is NaN — two `c` series
|
||||
// at one point — would answer one finite threshold per class instead.
|
||||
let matched = binary_tree_over_sources(BinaryOp::Add, raw_schema(), raw_schema(), 0, 1);
|
||||
let output = matched.root().output.clone();
|
||||
single_partition(
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(matched.root().clone()),
|
||||
right: Box::new(NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(instant_of(2, &grid)),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
}),
|
||||
matching: BinaryMatching {
|
||||
op: BinaryOp::Mul,
|
||||
return_bool: false,
|
||||
labels: MatchingLabels::Ignoring(Vec::new()),
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill: FillValues::default(),
|
||||
},
|
||||
},
|
||||
output,
|
||||
})
|
||||
.expect("a match under a scalar() broadcast is a legal tree"),
|
||||
);
|
||||
}
|
||||
|
||||
/// `x - 2 * x`: one source read at both operands, with a scalar multiplier on the right.
|
||||
///
|
||||
/// The subtraction and the multiplier together make the answer depend on which operand is which:
|
||||
@@ -6588,6 +6961,26 @@ fn histogram_over_sum_tree(group: Grouping, function: HistogramFunction) -> Nati
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// The same histogram over the same fold, with its parameters evaluated at every grid point instead
|
||||
/// of bound as constants: `histogram_quantile(<parameter>, sum by (...) (rate(x[5m])))`.
|
||||
fn histogram_parameterized_over_sum(
|
||||
histogram: NativeTree,
|
||||
parameters: Vec<NativeNode>,
|
||||
) -> NativeTree {
|
||||
let NativeExpr::Vector { input, op } = &histogram.root().expr else {
|
||||
unreachable!("`histogram_over_sum_tree` builds a literal histogram")
|
||||
};
|
||||
NativeTree::try_new(NativeNode {
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: input.clone(),
|
||||
op: op.clone(),
|
||||
parameters,
|
||||
},
|
||||
output: histogram.root().output.clone(),
|
||||
})
|
||||
.expect("a parameterized histogram over the same fold is a legal tree")
|
||||
}
|
||||
|
||||
fn histogram_routing() -> SeriesRouting {
|
||||
SeriesRouting::CanonicalLabelsExcept(
|
||||
["le", "__name__", "__type__", "__unit__"]
|
||||
@@ -6757,6 +7150,91 @@ fn histogram_split_allows_single_owner_routing_label_rewrites() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_split_carries_a_per_point_parameter() {
|
||||
let grid = grid();
|
||||
let base = histogram_over_sum_tree(
|
||||
Grouping::By(Arc::from([Arc::from("le")])),
|
||||
HistogramFunction::Quantile(0.99),
|
||||
);
|
||||
// `histogram_quantile(time() / 150, sum by (le) (rate(x[5m])))`, with the parameter bound the
|
||||
// way the compiler binds a per-point argument: the evaluation time broadcast over a constant.
|
||||
let eval_time = NativeNode {
|
||||
expr: NativeExpr::EvalTime { date: None },
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let divisor = NativeNode {
|
||||
expr: NativeExpr::Scalar(BoundScalar { value: 150.0 }),
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let parameter = NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(eval_time),
|
||||
right: Box::new(divisor),
|
||||
matching: BinaryMatching {
|
||||
op: BinaryOp::Div,
|
||||
return_bool: false,
|
||||
labels: MatchingLabels::Ignoring(Vec::new()),
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill: FillValues::default(),
|
||||
},
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let tree = histogram_parameterized_over_sum(base.clone(), vec![parameter.clone()]);
|
||||
let (partial, final_tree) = FusedSeries::split_aggregate(&tree, TIME)
|
||||
.unwrap_or_else(|error| panic!("a per-point parameter must split, not fail: {error}"))
|
||||
.unwrap_or_else(|| panic!("a per-point constant parameter must split: {tree}"));
|
||||
assert!(matches!(
|
||||
&partial.root().expr,
|
||||
NativeExpr::Aggregate { op, .. } if op.stage == AggregateStage::Partial
|
||||
));
|
||||
let NativeExpr::ParameterizedHistogram {
|
||||
input,
|
||||
op,
|
||||
parameters,
|
||||
} = &final_tree.root().expr
|
||||
else {
|
||||
panic!("the parameterized histogram must run above the Final aggregate: {final_tree}");
|
||||
};
|
||||
assert_eq!(
|
||||
&BoundVectorOp::Histogram(HistogramFunction::Quantile(0.99)),
|
||||
op
|
||||
);
|
||||
assert!(matches!(
|
||||
&input.expr,
|
||||
NativeExpr::Aggregate { op, .. } if op.stage == AggregateStage::Final
|
||||
));
|
||||
// The parameter is carried untouched: it reads no series, so it means the same above the cut.
|
||||
assert_eq!(vec![parameter], *parameters);
|
||||
|
||||
// A parameter that reads the series the split reads has no source above the cut, so it must
|
||||
// decline rather than be answered from the merged values: `histogram_quantile(scalar(sum(
|
||||
// rate(x[5m]))), sum by (le) (rate(x[5m])))`, whose parameter holds the window the prefix reads.
|
||||
let NativeExpr::Vector {
|
||||
input: aggregate, ..
|
||||
} = &base.root().expr
|
||||
else {
|
||||
unreachable!("`histogram_over_sum_tree` builds a literal histogram")
|
||||
};
|
||||
let NativeExpr::Aggregate { input: window, .. } = &aggregate.expr else {
|
||||
unreachable!("`histogram_over_sum_tree` folds the rate window")
|
||||
};
|
||||
let scalar_of_window = NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: window.clone(),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let reading = histogram_parameterized_over_sum(base, vec![scalar_of_window]);
|
||||
assert!(
|
||||
FusedSeries::split_aggregate(&reading, TIME)
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"`histogram_quantile(scalar(sum(rate(x[5m]))), sum by (le) (rate(x[5m])))` must decline: {reading}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_final_replaces_incompatible_state_router() {
|
||||
for (group, old_width, target_width, required_width, old_policy, reuse) in [
|
||||
|
||||
@@ -104,13 +104,25 @@ pub fn evaluate_histograms(
|
||||
schema: &Arc<SeriesSchema>,
|
||||
series: &[Series],
|
||||
) -> Result<Vec<Series>> {
|
||||
evaluate_histograms_with_annotations(function, schema, series, None)
|
||||
evaluate_histograms_with_annotations(function, schema, series, None, None)
|
||||
}
|
||||
|
||||
/// Per-grid-point arguments to a classic or native histogram function.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct HistogramParameters<'a> {
|
||||
pub quantile: Option<&'a [Option<f64>]>,
|
||||
pub lower: Option<&'a [Option<f64>]>,
|
||||
pub upper: Option<&'a [Option<f64>]>,
|
||||
}
|
||||
|
||||
/// Evaluates `function` with optional per-point scalar arguments, reporting what it drops to
|
||||
/// `collector`. A missing parameter value suppresses the output at that point, matching aggregate
|
||||
/// quantile's missing-q behavior.
|
||||
pub(crate) fn evaluate_histograms_with_annotations(
|
||||
function: HistogramFunction,
|
||||
schema: &Arc<SeriesSchema>,
|
||||
series: &[Series],
|
||||
parameters: Option<HistogramParameters<'_>>,
|
||||
collector: Option<&PromqlAnnotationCollector>,
|
||||
) -> Result<Vec<Series>> {
|
||||
// The groups in the order their first member arrived, so the result is in one order on every
|
||||
@@ -215,6 +227,31 @@ pub(crate) fn evaluate_histograms_with_annotations(
|
||||
_ => None,
|
||||
}
|
||||
});
|
||||
let argument = |values: Option<&[Option<f64>]>, bound: f64| -> Option<f64> {
|
||||
match values {
|
||||
Some(values) => values.get(point).copied().flatten(),
|
||||
None => Some(bound),
|
||||
}
|
||||
};
|
||||
let function = match function {
|
||||
HistogramFunction::Quantile(q) => {
|
||||
let Some(q) = argument(parameters.and_then(|p| p.quantile), q) else {
|
||||
*value = None;
|
||||
continue;
|
||||
};
|
||||
HistogramFunction::Quantile(q)
|
||||
}
|
||||
HistogramFunction::Fraction { lower, upper } => {
|
||||
let (Some(lower), Some(upper)) = (
|
||||
argument(parameters.and_then(|p| p.lower), lower),
|
||||
argument(parameters.and_then(|p| p.upper), upper),
|
||||
) else {
|
||||
*value = None;
|
||||
continue;
|
||||
};
|
||||
HistogramFunction::Fraction { lower, upper }
|
||||
}
|
||||
};
|
||||
// A native histogram and classic buckets of one label set conflict.
|
||||
*value = match (native, classic_buckets.is_empty()) {
|
||||
(Some(_), false) => {
|
||||
|
||||
@@ -24,8 +24,9 @@ use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis};
|
||||
|
||||
use super::float_histogram::{AddOutcome, FloatHistogram};
|
||||
use super::{
|
||||
HistogramAggregate, HistogramFunction, NativeHistogramFunction, aggregate_histograms,
|
||||
evaluate_histograms, histogram_quantiles, histogram_string, map_native_histogram,
|
||||
HistogramAggregate, HistogramFunction, HistogramParameters, NativeHistogramFunction,
|
||||
aggregate_histograms, evaluate_histograms, evaluate_histograms_with_annotations,
|
||||
histogram_quantiles, histogram_string, map_native_histogram,
|
||||
};
|
||||
use crate::grid_series::merge_same_labelset;
|
||||
use crate::grid_series::reference::{
|
||||
@@ -102,6 +103,110 @@ fn assert_outcome(
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classic_histograms_accept_per_point_parameters() {
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 1_000, 2).unwrap()).unwrap());
|
||||
let schema = Arc::new(SeriesSchema {
|
||||
fields: vec![FieldSpec {
|
||||
name: Arc::from("value"),
|
||||
}]
|
||||
.into(),
|
||||
eval_grid: Some(grid),
|
||||
});
|
||||
let mut input = Vec::new();
|
||||
for (le, counts) in [
|
||||
("1", [5.0_f64, 5.0]),
|
||||
("2", [9.0_f64, 9.0]),
|
||||
("+Inf", [10.0_f64, 10.0]),
|
||||
] {
|
||||
let labels = Labels::from([
|
||||
("le".to_string(), le.to_string()),
|
||||
("__name__".to_string(), "bucket".to_string()),
|
||||
]);
|
||||
input.push(
|
||||
mixed_series(
|
||||
&schema,
|
||||
&labels,
|
||||
&BTreeMap::from([(0, counts[0].to_bits()), (1, counts[1].to_bits())]),
|
||||
&BTreeMap::new(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let q = [Some(0.5), Some(0.9)];
|
||||
let result = evaluate_histograms_with_annotations(
|
||||
HistogramFunction::Quantile(0.0),
|
||||
&schema,
|
||||
&input,
|
||||
Some(HistogramParameters {
|
||||
quantile: Some(&q),
|
||||
lower: None,
|
||||
upper: None,
|
||||
}),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let points = points_by_labels(result);
|
||||
let actual = points.values().next().unwrap();
|
||||
let at = |q| {
|
||||
let r = evaluate_histograms(HistogramFunction::Quantile(q), &schema, &input).unwrap();
|
||||
points_by_labels(r).values().next().unwrap().clone()
|
||||
};
|
||||
assert_eq!(actual[&0], at(0.5)[&0]);
|
||||
assert_eq!(actual[&1], at(0.9)[&1]);
|
||||
assert_ne!(actual[&0], actual[&1]);
|
||||
for q in [f64::NAN, -1.0, 2.0] {
|
||||
let dynamic = evaluate_histograms_with_annotations(
|
||||
HistogramFunction::Quantile(0.0),
|
||||
&schema,
|
||||
&input,
|
||||
Some(HistogramParameters {
|
||||
quantile: Some(&[Some(q), Some(q)]),
|
||||
lower: None,
|
||||
upper: None,
|
||||
}),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let constant =
|
||||
evaluate_histograms(HistogramFunction::Quantile(q), &schema, &input).unwrap();
|
||||
assert_eq!(points_by_labels(dynamic), points_by_labels(constant));
|
||||
}
|
||||
let lower = [Some(0.5), Some(1.5)];
|
||||
let upper = [Some(1.5), Some(2.5)];
|
||||
let dynamic = evaluate_histograms_with_annotations(
|
||||
HistogramFunction::Fraction {
|
||||
lower: 0.0,
|
||||
upper: 1.0,
|
||||
},
|
||||
&schema,
|
||||
&input,
|
||||
Some(HistogramParameters {
|
||||
quantile: None,
|
||||
lower: Some(&lower),
|
||||
upper: Some(&upper),
|
||||
}),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let points = points_by_labels(dynamic);
|
||||
for i in 0..2 {
|
||||
let constant = evaluate_histograms(
|
||||
HistogramFunction::Fraction {
|
||||
lower: lower[i].unwrap(),
|
||||
upper: upper[i].unwrap(),
|
||||
},
|
||||
&schema,
|
||||
&input,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
points.values().next().unwrap()[&i],
|
||||
points_by_labels(constant).values().next().unwrap()[&i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classic_histograms_match_prometheus() {
|
||||
let cases = golden_cases(include_str!("classic_cases.txt"));
|
||||
|
||||
@@ -220,6 +220,36 @@ fn rewrite(node: &NativeNode, source_tags: &[Vec<String>]) -> DfResult<NativeNod
|
||||
NativeExpr::VectorOf { input } => NativeExpr::VectorOf {
|
||||
input: Box::new(rewrite(input, source_tags)?),
|
||||
},
|
||||
// `scalar()` answers from the whole vector: a series that arrives in pieces would be seen
|
||||
// as several samples, so the pieces are collected where they still hold their samples.
|
||||
NativeExpr::ScalarOf { input } => NativeExpr::ScalarOf {
|
||||
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
|
||||
},
|
||||
// `topk` ranks the whole vector, so it counts canonical series; its parameter is a scalar,
|
||||
// rewritten but never collected - a scalar carries no label set.
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input,
|
||||
op,
|
||||
parameter,
|
||||
} => NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
|
||||
op: op.clone(),
|
||||
parameter: Box::new(rewrite(parameter, source_tags)?),
|
||||
},
|
||||
// A histogram function folds the bucket series of one histogram, so overlapping fragments
|
||||
// must not reach it; its parameters are scalars and only need rewriting.
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input,
|
||||
op,
|
||||
parameters,
|
||||
} => NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(canonical(rewrite(input, source_tags)?, source_tags)),
|
||||
op: op.clone(),
|
||||
parameters: parameters
|
||||
.iter()
|
||||
.map(|parameter| rewrite(parameter, source_tags))
|
||||
.collect::<DfResult<Vec<_>>>()?,
|
||||
},
|
||||
};
|
||||
Ok(NativeNode {
|
||||
expr,
|
||||
|
||||
@@ -301,6 +301,9 @@ fn merges(tree: &NativeTree) -> usize {
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::ParameterizedMap { input, .. }
|
||||
| NativeExpr::ParameterizedTopK { input, .. }
|
||||
| NativeExpr::ParameterizedHistogram { input, .. }
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
|
||||
@@ -100,6 +100,25 @@ fn describe(node: &NativeNode) -> String {
|
||||
format!("{}() on grid {}", date.name(), grid_of(node))
|
||||
}
|
||||
NativeExpr::VectorOf { .. } => "vector()".to_string(),
|
||||
NativeExpr::ScalarOf { .. } => "scalar()".to_string(),
|
||||
NativeExpr::ParameterizedTopK { op, .. } => match op {
|
||||
BoundVectorOp::TopK { top: true, .. } => "parameterized topk".to_string(),
|
||||
BoundVectorOp::TopK { top: false, .. } => "parameterized bottomk".to_string(),
|
||||
_ => "parameterized topk".to_string(),
|
||||
},
|
||||
NativeExpr::ParameterizedHistogram { op, parameters, .. } => {
|
||||
format!(
|
||||
"{} ({} parameters)",
|
||||
match op {
|
||||
BoundVectorOp::Histogram(HistogramFunction::Quantile(_)) =>
|
||||
"histogram_quantile",
|
||||
BoundVectorOp::Histogram(HistogramFunction::Fraction { .. }) =>
|
||||
"histogram_fraction",
|
||||
_ => "histogram",
|
||||
},
|
||||
parameters.len()
|
||||
)
|
||||
}
|
||||
NativeExpr::ParameterizedMap { op, .. } => match op {
|
||||
ParameterizedOp::Round => "round".to_string(),
|
||||
ParameterizedOp::Clamp => "clamp".to_string(),
|
||||
@@ -409,7 +428,10 @@ fn kind(expr: &NativeExpr) -> &'static str {
|
||||
NativeExpr::Binary { .. } => "Binary",
|
||||
NativeExpr::EvalTime { .. } => "EvalTime",
|
||||
NativeExpr::VectorOf { .. } => "VectorOf",
|
||||
NativeExpr::ScalarOf { .. } => "ScalarOf",
|
||||
NativeExpr::ParameterizedMap { .. } => "ParameterizedMap",
|
||||
NativeExpr::ParameterizedTopK { .. } => "ParameterizedTopK",
|
||||
NativeExpr::ParameterizedHistogram { .. } => "ParameterizedHistogram",
|
||||
NativeExpr::Set { .. } => "Set",
|
||||
NativeExpr::Info { .. } => "Info",
|
||||
}
|
||||
@@ -437,7 +459,22 @@ fn display_node(
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::VectorOf { input } => display_node(f, input, depth + 1, "input")?,
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => display_node(f, input, depth + 1, "input")?,
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
display_node(f, input, depth + 1, "input")?;
|
||||
for parameter in parameters {
|
||||
display_node(f, parameter, depth + 1, "parameter")?;
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
display_node(f, input, depth + 1, "input")?;
|
||||
display_node(f, parameter, depth + 1, "parameter")?;
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -125,6 +125,19 @@ pub fn fused_output_tags(node: &NativeNode, source_tags: &[Vec<String>]) -> Opti
|
||||
// Its parameters are scalars, which carry no labels, so what the node reports are its
|
||||
// input's labels.
|
||||
NativeExpr::ParameterizedMap { input, .. } => fused_output_tags(input, source_tags),
|
||||
// `scalar()` reports one value and no labels, like the scalar it produces.
|
||||
NativeExpr::ScalarOf { .. } => Some(Vec::new()),
|
||||
// Its parameter is a scalar, which carries no labels, and ranking keeps each selected
|
||||
// series' own labels.
|
||||
NativeExpr::ParameterizedTopK { input, .. } => fused_output_tags(input, source_tags),
|
||||
// A parameterized histogram function folds the bucket series that share the other labels
|
||||
// into one series and drops `le` along with the metadata labels, exactly as its bound
|
||||
// counterpart does. Its parameters are scalars and carry no labels.
|
||||
NativeExpr::ParameterizedHistogram { input, .. } => {
|
||||
let mut tags = fused_output_tags(input, source_tags)?;
|
||||
tags.retain(|tag| tag != LE_LABEL && !PROMQL_METADATA_LABELS.contains(&tag.as_str()));
|
||||
Some(tags)
|
||||
}
|
||||
// An aggregate reports the grouping labels its input carries: `by(...)` keeps exactly the
|
||||
// ones it names, and `without(...)` keeps the rest and drops the metric name, as Prometheus
|
||||
// does.
|
||||
@@ -488,6 +501,14 @@ impl NativeNode {
|
||||
let left_kind = kind_of(left)?;
|
||||
(left_kind == kind_of(right)?).then_some(left_kind)
|
||||
}
|
||||
// `scalar()` answers one float per grid point, whatever its input holds.
|
||||
NativeExpr::ScalarOf { .. } => Some(FieldKind::Float),
|
||||
// A ranking hands on the series it selected, so the field keeps the kind it read; its
|
||||
// parameter is a scalar, which carries no samples of its own.
|
||||
NativeExpr::ParameterizedTopK { input, .. } => kind_of(input),
|
||||
// `histogram_quantile` and `histogram_fraction` reduce a histogram to one float per
|
||||
// sample whatever their arguments are, constant or per point.
|
||||
NativeExpr::ParameterizedHistogram { .. } => Some(FieldKind::Float),
|
||||
// An aggregate folds float state only: the admission gate keeps a histogram from
|
||||
// entering one, so whatever a histogram statement aggregates answers floats. Naming it
|
||||
// here is what lets `sum(count_over_time(h[5m]))` - a float built from histograms -
|
||||
@@ -787,6 +808,13 @@ pub struct BoundSetOp {
|
||||
}
|
||||
|
||||
/// What a node computes. Every variant holds a bound operation, never a name to resolve.
|
||||
///
|
||||
/// The variants are numbered by their position, and postcard writes that number, so these indices
|
||||
/// are a wire contract: a version 3 [`crate::extension_plan::FusedSeries`] envelope carries the
|
||||
/// whole [`NativeTree`] rather than a prefix of it, and a variant inserted anywhere but the end
|
||||
/// renames every variant after it for a reader of an earlier build. New variants are therefore
|
||||
/// appended after `Info`, and `native_expr_variant_indices_are_a_wire_contract` in the tests below
|
||||
/// pins the numbers.
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
|
||||
pub enum NativeExpr {
|
||||
/// A scan or an exchange boundary.
|
||||
@@ -860,6 +888,24 @@ pub enum NativeExpr {
|
||||
info: Box<NativeNode>,
|
||||
selector: InfoSelector,
|
||||
},
|
||||
/// `scalar(v)`: at each grid point, the sole sample value or NaN when the vector has zero or
|
||||
/// more than one sample.
|
||||
ScalarOf { input: Box<NativeNode> },
|
||||
/// `topk`/`bottomk` whose `k` is evaluated at every grid point instead of bound as a constant.
|
||||
///
|
||||
/// `op` is still the `TopK` the constant path binds; its `k` is unused here, because the value
|
||||
/// this node selects with is the parameter's, read once per point.
|
||||
ParameterizedTopK {
|
||||
input: Box<NativeNode>,
|
||||
op: BoundVectorOp,
|
||||
parameter: Box<NativeNode>,
|
||||
},
|
||||
/// A classic histogram vector operation whose arguments vary on the output grid.
|
||||
ParameterizedHistogram {
|
||||
input: Box<NativeNode>,
|
||||
op: BoundVectorOp,
|
||||
parameters: Vec<NativeNode>,
|
||||
},
|
||||
}
|
||||
|
||||
/// One node of a bound tree.
|
||||
@@ -917,9 +963,15 @@ pub fn duplicates_canonical_labels(node: &NativeNode, source_tags: &[Vec<String>
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::ParameterizedMap { input, .. } => {
|
||||
| NativeExpr::ParameterizedMap { input, .. }
|
||||
| NativeExpr::ParameterizedTopK { input, .. } => {
|
||||
duplicates_canonical_labels(input, source_tags)
|
||||
}
|
||||
// `scalar()` answers one value, which names no label set at all.
|
||||
NativeExpr::ScalarOf { .. } => false,
|
||||
// The histogram functions fold the bucket series of one histogram into one series, so two
|
||||
// pieces of one label set cannot survive them.
|
||||
NativeExpr::ParameterizedHistogram { .. } => false,
|
||||
// A rewrite writes a caller-chosen destination label, so two series can end under one set -
|
||||
// but only if that label was one of the input's, because writing a label no series carries
|
||||
// leaves every series' identity distinct. `map_writes_existing_label` is that question, and
|
||||
@@ -1018,8 +1070,17 @@ pub fn reads_across_series(node: &NativeNode) -> bool {
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::ParameterizedMap { input, .. }
|
||||
| NativeExpr::Timestamp { input } => reads_across_series(input),
|
||||
// A parameterized operation reads its parameters too, and one of them can read across series
|
||||
// as well: `clamp_min(b, scalar(a))` bounds each series of `b` by a value that only exists
|
||||
// once the whole of `a` is gathered, so the node reads across series wherever its parameter
|
||||
// does.
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => reads_across_series(input) || parameters.iter().any(reads_across_series),
|
||||
// `scalar()` needs the whole vector: which series carries the one value at a point decides
|
||||
// the answer, so the node reads across series.
|
||||
NativeExpr::ScalarOf { .. } => true,
|
||||
NativeExpr::Binary { left, right, .. } => {
|
||||
let scalar = |side: &NativeNode| matches!(side.output, NativeType::Scalar(_));
|
||||
if scalar(left) || scalar(right) {
|
||||
@@ -1029,6 +1090,9 @@ pub fn reads_across_series(node: &NativeNode) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
// A parameterized operation reads its parameter too, and a `k` that changes per point
|
||||
// decides which series survive.
|
||||
NativeExpr::ParameterizedTopK { .. } | NativeExpr::ParameterizedHistogram { .. } => true,
|
||||
NativeExpr::Aggregate { .. }
|
||||
| NativeExpr::Vector { .. }
|
||||
| NativeExpr::Set { .. }
|
||||
@@ -1128,6 +1192,7 @@ fn ensure_native_node_depth(root: &NativeNode) -> Result<()> {
|
||||
}
|
||||
match &node.expr {
|
||||
NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Window { input, .. }
|
||||
@@ -1135,8 +1200,17 @@ fn ensure_native_node_depth(root: &NativeNode) -> Result<()> {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => visit(input, depth + 1),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
visit(input, depth + 1)?;
|
||||
visit(parameter, depth + 1)
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
visit(input, depth + 1)?;
|
||||
for parameter in parameters {
|
||||
@@ -1176,6 +1250,18 @@ impl NativeNode {
|
||||
return Err(invalid("a scalar node does not produce a scalar"));
|
||||
}
|
||||
}
|
||||
NativeExpr::ScalarOf { input } => {
|
||||
input.validate()?;
|
||||
let Some((_, input_grid)) = evaluated_vector(&input.output) else {
|
||||
return Err(invalid("scalar() consumes an evaluated vector"));
|
||||
};
|
||||
let NativeType::Scalar(output_grid) = &self.output else {
|
||||
return Err(invalid("scalar() does not produce a scalar"));
|
||||
};
|
||||
if output_grid.spec() != input_grid.spec() {
|
||||
return Err(invalid("scalar() output is not on its input's grid"));
|
||||
}
|
||||
}
|
||||
NativeExpr::VectorOf { input } => {
|
||||
input.validate()?;
|
||||
let NativeType::Scalar(input_grid) = &input.output else {
|
||||
@@ -1313,6 +1399,72 @@ impl NativeNode {
|
||||
ensure_same_fields(&input.output, &self.output, "a parameterized map")?;
|
||||
ensure_grid(&self.output, input_grid, "a parameterized map")?;
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input,
|
||||
op,
|
||||
parameter,
|
||||
} => {
|
||||
input.validate()?;
|
||||
parameter.validate()?;
|
||||
if !matches!(op, BoundVectorOp::TopK { .. }) {
|
||||
return Err(invalid("a parameterized topk operation is not TopK"));
|
||||
}
|
||||
let Some((_, output_grid)) = evaluated_vector(&self.output) else {
|
||||
return Err(invalid(
|
||||
"a parameterized topk does not produce an evaluated vector",
|
||||
));
|
||||
};
|
||||
if !matches!(¶meter.output, NativeType::Scalar(grid) if grid.spec() == output_grid.spec())
|
||||
{
|
||||
return Err(invalid(
|
||||
"a topk parameter is not a scalar on its output grid",
|
||||
));
|
||||
}
|
||||
let Some((_, input_grid)) = evaluated_vector(&input.output) else {
|
||||
return Err(invalid("a parameterized topk consumes a non-vector"));
|
||||
};
|
||||
ensure_same_fields(&input.output, &self.output, "a vector operation")?;
|
||||
ensure_grid(&input.output, input_grid, "a vector operation")?;
|
||||
ensure_grid(&self.output, input_grid, "a vector operation")?;
|
||||
}
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input,
|
||||
op,
|
||||
parameters,
|
||||
} => {
|
||||
input.validate()?;
|
||||
if !matches!(op, BoundVectorOp::Histogram(_)) {
|
||||
return Err(invalid(
|
||||
"a parameterized histogram operation is not Histogram",
|
||||
));
|
||||
}
|
||||
let expected = match op {
|
||||
BoundVectorOp::Histogram(HistogramFunction::Quantile(_)) => 1,
|
||||
BoundVectorOp::Histogram(HistogramFunction::Fraction { .. }) => 2,
|
||||
_ => {
|
||||
return Err(invalid(
|
||||
"parameterized histogram operation is not Histogram",
|
||||
));
|
||||
}
|
||||
};
|
||||
if parameters.len() != expected {
|
||||
return Err(invalid("parameterized histogram has the wrong arity"));
|
||||
}
|
||||
let Some((_, input_grid)) = evaluated_vector(&input.output) else {
|
||||
return Err(invalid("a parameterized histogram consumes a non-vector"));
|
||||
};
|
||||
for parameter in parameters {
|
||||
parameter.validate()?;
|
||||
if !matches!(¶meter.output, NativeType::Scalar(grid) if grid.spec() == input_grid.spec())
|
||||
{
|
||||
return Err(invalid(
|
||||
"a histogram parameter is not a scalar on its input grid",
|
||||
));
|
||||
}
|
||||
}
|
||||
ensure_same_fields(&input.output, &self.output, "a vector operation")?;
|
||||
ensure_grid(&self.output, input_grid, "a vector operation")?;
|
||||
}
|
||||
NativeExpr::Vector { input, .. } => {
|
||||
input.validate()?;
|
||||
// A vector operation keeps the input's fields and grid, so it consumes and produces
|
||||
@@ -1697,6 +1849,13 @@ pub fn unifies_its_own_contributions(node: &NativeNode) -> Option<bool> {
|
||||
// A map can rewrite labels, so two series can end under one label set and the runtime
|
||||
// records what it wrote in an `Emitted` table.
|
||||
NativeExpr::Map { .. } | NativeExpr::ParameterizedMap { .. } => Some(false),
|
||||
// `topk` hands back the series it read with their labels, so the input's answer is its own -
|
||||
// exactly as for the bound operation.
|
||||
NativeExpr::ParameterizedTopK { input, .. } => unifies_its_own_contributions(input),
|
||||
// A histogram function groups its rows by label set itself, so it emits one series per set.
|
||||
NativeExpr::ParameterizedHistogram { .. } => Some(true),
|
||||
// `scalar()` answers one value per point, which is not several series.
|
||||
NativeExpr::ScalarOf { .. } => Some(true),
|
||||
// A timestamp reports a position under the metric name dropped.
|
||||
NativeExpr::Timestamp { .. } => Some(false),
|
||||
NativeExpr::Binary {
|
||||
|
||||
@@ -252,6 +252,47 @@ fn add() -> BinaryMatching {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameterized_topk_validates_scalar_and_grid() {
|
||||
let out = grid(0, 60_000, 4);
|
||||
let vector = map_over(window_on(input(), None, Arc::clone(&out)));
|
||||
let make = |parameter, op| NativeNode {
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(vector.clone()),
|
||||
op,
|
||||
parameter: Box::new(parameter),
|
||||
},
|
||||
output: vector.output.clone(),
|
||||
};
|
||||
let op = BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
};
|
||||
assert!(
|
||||
make(scalar_on(1.0, Arc::clone(&out)), op.clone())
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
make(scalar_on(1.0, grid(0, 30_000, 8)), op.clone())
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(make(vector.clone(), op.clone()).validate().is_err());
|
||||
assert!(
|
||||
make(
|
||||
scalar_on(1.0, out),
|
||||
BoundVectorOp::LimitK {
|
||||
k: 1.0,
|
||||
group: Grouping::By(Arc::from([]))
|
||||
}
|
||||
)
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
/// A compiled tree is accepted when every node's contract holds.
|
||||
#[test]
|
||||
fn a_window_over_raw_samples_is_valid() {
|
||||
@@ -951,6 +992,14 @@ fn a_constant_operand_reads_one_series_at_a_time() {
|
||||
aggregate,
|
||||
scalar_on(100.0, Arc::clone(&shared))
|
||||
)));
|
||||
// `scalar(rate(v0[5m]))` must inspect all the series at each point.
|
||||
let scalar_of = NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(vector()),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&shared)),
|
||||
};
|
||||
assert!(reads_across_series(&scalar_of));
|
||||
// `rate(v0[5m]) + rate(v0[5m])` matches two vectors by label.
|
||||
assert!(reads_across_series(&binary(vector(), vector())));
|
||||
// `1 + 2` answers one value for every series there is.
|
||||
@@ -2045,6 +2094,173 @@ fn native_tree_depth_limit_holds_across_lifecycle_on_named_worker_stack() {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tree_depth_limit_checks_dynamic_children() {
|
||||
use super::{MAX_NATIVE_NODE_DEPTH, ParameterizedOp, ensure_native_node_depth};
|
||||
use crate::histogram_eval::HistogramFunction;
|
||||
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let mut deep = input_on(Arc::clone(&grid));
|
||||
for _ in 0..MAX_NATIVE_NODE_DEPTH - 1 {
|
||||
deep = map_over(deep);
|
||||
}
|
||||
let scalar_of = |input| NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(input),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let scalar = scalar_on(1.0, Arc::clone(&grid));
|
||||
let input = input_on(Arc::clone(&grid));
|
||||
let topk = |input: NativeNode, parameter| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
},
|
||||
};
|
||||
let histogram = |input: NativeNode, parameters| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Fraction {
|
||||
lower: 0.0,
|
||||
upper: 0.0,
|
||||
}),
|
||||
parameters,
|
||||
},
|
||||
};
|
||||
let map = |input: NativeNode, parameters| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedMap {
|
||||
input: Box::new(input),
|
||||
op: ParameterizedOp::Clamp,
|
||||
parameters,
|
||||
},
|
||||
};
|
||||
assert!(ensure_native_node_depth(&deep).is_ok());
|
||||
for root in [
|
||||
scalar_of(deep.clone()),
|
||||
topk(deep.clone(), scalar.clone()),
|
||||
topk(input.clone(), scalar_of(deep.clone())),
|
||||
histogram(deep.clone(), vec![scalar.clone(), scalar.clone()]),
|
||||
histogram(input.clone(), vec![scalar.clone(), scalar_of(deep.clone())]),
|
||||
map(deep.clone(), vec![scalar.clone(), scalar.clone()]),
|
||||
map(input, vec![scalar, scalar_of(deep)]),
|
||||
] {
|
||||
assert!(matches!(
|
||||
NativeTree::try_new(root),
|
||||
Err(DataFusionError::NotImplemented(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/// The index postcard writes for a [`NativeExpr`] variant is a wire contract, not an implementation
|
||||
/// detail: postcard encodes a variant as its index, and a version 3 `FusedSeries` envelope carries
|
||||
/// the whole tree rather than a prefix of it, so a variant inserted anywhere but the end renames
|
||||
/// every variant after it for a reader of an earlier build.
|
||||
///
|
||||
/// The numbers here are the ones already written to the wire. New variants are appended after
|
||||
/// `Info`; every variant above them keeps the index below.
|
||||
#[test]
|
||||
fn native_expr_variant_indices_are_a_wire_contract() {
|
||||
use crate::histogram_eval::HistogramFunction;
|
||||
|
||||
// A node carries its expression first, so this is also the tag of any node or tree holding it.
|
||||
fn tag(expr: &NativeExpr) -> u8 {
|
||||
let encoded = postcard::to_stdvec(expr).expect("a native expression encodes");
|
||||
*encoded.first().expect("a variant tag is the first byte")
|
||||
}
|
||||
|
||||
let on_grid = || grid(0, 60_000, 1);
|
||||
let vector = || map_over(input_on(on_grid()));
|
||||
let scalar = || scalar_on(1.0, on_grid());
|
||||
let cases = [
|
||||
(
|
||||
NativeExpr::Input(BoundInput {
|
||||
source_index: 0,
|
||||
output: raw(),
|
||||
ordering: SeriesOrdering::KeyThenTime,
|
||||
}),
|
||||
0u8,
|
||||
"Input",
|
||||
),
|
||||
(NativeExpr::Scalar(BoundScalar { value: 1.0 }), 1, "Scalar"),
|
||||
(NativeExpr::EvalTime { date: None }, 2, "EvalTime"),
|
||||
(
|
||||
NativeExpr::VectorOf {
|
||||
input: Box::new(scalar()),
|
||||
},
|
||||
3,
|
||||
"VectorOf",
|
||||
),
|
||||
(
|
||||
NativeExpr::Merge {
|
||||
input: Box::new(vector()),
|
||||
},
|
||||
5,
|
||||
"Merge",
|
||||
),
|
||||
(
|
||||
NativeExpr::Timestamp {
|
||||
input: Box::new(vector()),
|
||||
},
|
||||
9,
|
||||
"Timestamp",
|
||||
),
|
||||
(
|
||||
NativeExpr::Aggregate {
|
||||
input: Box::new(vector()),
|
||||
op: BoundAggregate {
|
||||
kind: AggregateKind::Sum,
|
||||
group: Grouping::By(Arc::new(["job".into()])),
|
||||
stage: AggregateStage::Complete,
|
||||
},
|
||||
},
|
||||
10,
|
||||
"Aggregate",
|
||||
),
|
||||
// The dynamic forms are the last three, which is what keeps every index above them.
|
||||
(
|
||||
NativeExpr::ScalarOf {
|
||||
input: Box::new(vector()),
|
||||
},
|
||||
15,
|
||||
"ScalarOf",
|
||||
),
|
||||
(
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(vector()),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 1.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(scalar()),
|
||||
},
|
||||
16,
|
||||
"ParameterizedTopK",
|
||||
),
|
||||
(
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(vector()),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.9)),
|
||||
parameters: vec![scalar()],
|
||||
},
|
||||
17,
|
||||
"ParameterizedHistogram",
|
||||
),
|
||||
];
|
||||
for (expr, index, variant) in cases {
|
||||
assert_eq!(index, tag(&expr), "{variant} changed the index it encodes");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tree_postcard_float_fidelity_round_trip() {
|
||||
for bits in [
|
||||
|
||||
@@ -65,8 +65,10 @@ pub(crate) fn histogram_group_is_single(group: &Grouping) -> bool {
|
||||
/// are owned. Histogram functions can also run there when the state router co-locates every bucket
|
||||
/// and every histogram whose output labels collide. A label rewrite may change that key, so it is
|
||||
/// only admitted alongside a partitioned histogram when it cannot change a label in the cut's
|
||||
/// routing key. A single-owner histogram needs no such restriction. `topk`, a second aggregate and
|
||||
/// vector-vector matching still need a whole-vector boundary.
|
||||
/// routing key. A single-owner histogram needs no such restriction. A histogram parameter is
|
||||
/// answered above the cut, so it has to read no series: one that does would be silently replaced by
|
||||
/// the merged values. `topk`, a second aggregate and vector-vector matching still need a whole-vector
|
||||
/// boundary.
|
||||
///
|
||||
/// An instant selection or window above the cut cannot be applied to the merged values under the
|
||||
/// tree's current binding: the cut output is evaluated, whereas these nodes consume raw samples.
|
||||
@@ -176,18 +178,70 @@ pub(crate) fn split(
|
||||
};
|
||||
Ok(Some(inner))
|
||||
}
|
||||
// The same histogram operation with a parameter evaluated at every grid point instead of
|
||||
// bound as a constant. The parameter is a per-point scalar - an expression of `time()` and
|
||||
// literals - so it reads no series and means the same thing above the merged values, and
|
||||
// the suffix carries it unchanged.
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input,
|
||||
op,
|
||||
parameters,
|
||||
} => {
|
||||
let Some(mut inner) = split(input, fields, time_index)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// The same two refusals as the literal form: successive calls may drop different labels,
|
||||
// so only the first call's partition key is known, and a rewrite may change that key.
|
||||
if inner.histogram {
|
||||
return Err(declined("nested histograms above a split aggregate"));
|
||||
}
|
||||
if inner.label_rewrite && !histogram_group_is_single(&inner.op.group) {
|
||||
return Err(declined(
|
||||
"a routing-label rewrite around a partitioned histogram",
|
||||
));
|
||||
}
|
||||
// A parameter is answered at its own grid point rather than from the merged values, so
|
||||
// one that reads the series the prefix reads has no source above the cut: applying it
|
||||
// there would silently answer it from the merged state instead.
|
||||
if parameters
|
||||
.iter()
|
||||
.any(|parameter| !suffix_carries(parameter))
|
||||
{
|
||||
return Err(declined(
|
||||
"a histogram parameter that reads the series the split reads",
|
||||
));
|
||||
}
|
||||
inner.histogram = true;
|
||||
inner.suffix = NativeNode {
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(inner.suffix),
|
||||
op: op.clone(),
|
||||
parameters: parameters.clone(),
|
||||
},
|
||||
output: node.output.clone(),
|
||||
};
|
||||
Ok(Some(inner))
|
||||
}
|
||||
// TopK ranks the whole vector, not one histogram or aggregation group at a time.
|
||||
NativeExpr::Vector {
|
||||
op: BoundVectorOp::TopK { .. },
|
||||
..
|
||||
} => Err(declined("a whole-vector operation above a split aggregate")),
|
||||
NativeExpr::Merge { .. } => Err(declined("an aggregate split across a canonical merge")),
|
||||
// These read values the merged state does not answer: an instant selection or window reads
|
||||
// raw samples, `scalar`/`vector` answers one value or one series instead of the vector, and a
|
||||
// parameterized map reads the whole vector at a point. `ParameterizedTopK` declines here
|
||||
// exactly like the literal `topk` above does: a per-point `k` ranks the whole vector just as
|
||||
// a bound one does, so it needs the same whole-vector boundary. Admitting the one and
|
||||
// refusing the other would be an inconsistency rather than a capability.
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ParameterizedMap { input, .. } => match split(input, fields, time_index)? {
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::ParameterizedMap { input, .. }
|
||||
| NativeExpr::ParameterizedTopK { input, .. } => match split(input, fields, time_index)? {
|
||||
Some(_) => Err(declined(
|
||||
"an aggregate under an operation that reads evaluated values",
|
||||
)),
|
||||
@@ -286,10 +340,17 @@ pub(crate) fn prefix_fields(tree: &NativeTree) -> DfResult<Vec<FieldSpec>> {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::VectorOf { input } => raw_fields(input),
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => raw_fields(input),
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => raw_fields(input).or_else(|| parameters.iter().find_map(raw_fields)),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => raw_fields(input).or_else(|| raw_fields(parameter)),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => raw_fields(input).or_else(|| parameters.iter().find_map(raw_fields)),
|
||||
NativeExpr::Binary { left, right, .. }
|
||||
| NativeExpr::Set { left, right, .. }
|
||||
| NativeExpr::Info {
|
||||
@@ -481,8 +542,8 @@ impl Split {
|
||||
/// Puts `merged` at the cut and returns the suffix above it.
|
||||
///
|
||||
/// A suffix is the spine of point-wise maps, constant-sided binaries and at most one histogram
|
||||
/// operation between the cut and the root, with scalar constants preserved. Other whole-vector
|
||||
/// operations are refused by `split`.
|
||||
/// operation between the cut and the root, with scalar constants and the evaluation time preserved.
|
||||
/// Other whole-vector operations are refused by `split`.
|
||||
fn enter_suffix(node: NativeNode, merged: &NativeNode) -> DfResult<NativeNode> {
|
||||
let NativeNode { expr, output } = node;
|
||||
let rejected = || {
|
||||
@@ -490,8 +551,10 @@ fn enter_suffix(node: NativeNode, merged: &NativeNode) -> DfResult<NativeNode> {
|
||||
};
|
||||
let expr = match expr {
|
||||
NativeExpr::Input(_) => return Ok(merged.clone()),
|
||||
// A constant carries no state: the suffix keeps it as it is, beside the merged values.
|
||||
NativeExpr::Scalar(_) => {
|
||||
// A constant carries no state: the suffix keeps it as it is, beside the merged values. The
|
||||
// evaluation time is such a constant - it answers the grid's own timestamps, and the suffix
|
||||
// runs on the statement's grid, so it answers the same per-point value there.
|
||||
NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => {
|
||||
return Ok(NativeNode { expr, output });
|
||||
}
|
||||
NativeExpr::Map { input, op } => NativeExpr::Map {
|
||||
@@ -505,6 +568,18 @@ fn enter_suffix(node: NativeNode, merged: &NativeNode) -> DfResult<NativeNode> {
|
||||
input: Box::new(enter_suffix(*input, merged)?),
|
||||
op,
|
||||
},
|
||||
// The parameters are per-point scalars that read no series - `split` refuses the split
|
||||
// otherwise - so they mean the same above the merged values and are handed over untouched
|
||||
// while only the input is rebuilt.
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input,
|
||||
op,
|
||||
parameters,
|
||||
} => NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(enter_suffix(*input, merged)?),
|
||||
op,
|
||||
parameters,
|
||||
},
|
||||
NativeExpr::Binary {
|
||||
left,
|
||||
right,
|
||||
@@ -518,3 +593,30 @@ fn enter_suffix(node: NativeNode, merged: &NativeNode) -> DfResult<NativeNode> {
|
||||
};
|
||||
Ok(NativeNode { expr, output })
|
||||
}
|
||||
|
||||
/// Whether [`enter_suffix`] rebuilds this subtree without reading `merged`.
|
||||
///
|
||||
/// This is the question the histogram arm of [`split`] asks of every parameter before it is carried
|
||||
/// above the cut: a parameter that `enter_suffix` rebuilds from constants alone means the same thing
|
||||
/// there, and one that reads the series has no source above the cut. Keeping the answer exactly the
|
||||
/// set of shapes `enter_suffix` builds is what keeps a misplaced tree a decline rather than an
|
||||
/// `Internal` error: a subtree that claims to carry but does not would reach `rejected()`.
|
||||
fn suffix_carries(node: &NativeNode) -> bool {
|
||||
match &node.expr {
|
||||
// Constants: they answer their own values on the statement's grid.
|
||||
NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => true,
|
||||
// Point-wise spines over something that carries: the map and the histogram application
|
||||
// reach each series the way they are rebuilt.
|
||||
NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Vector {
|
||||
input,
|
||||
op: BoundVectorOp::Histogram(_),
|
||||
} => suffix_carries(input),
|
||||
// A binary operation carries when both sides do: a constant broadcast over a constant is
|
||||
// still a constant.
|
||||
NativeExpr::Binary { left, right, .. } => suffix_carries(left) && suffix_carries(right),
|
||||
// Everything else reads a source: the cut's own values (`Input`), the whole vector at a
|
||||
// point (`ScalarOf`, a vector operation), or the series a parameter reads.
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,14 +69,23 @@ pub fn topk(
|
||||
grouping: &Grouping,
|
||||
schema: &SeriesSchema,
|
||||
series: &[Series],
|
||||
) -> Result<Vec<Series>> {
|
||||
let k = k_param(k, series.len())?;
|
||||
topk_at(|_| Ok(k), top, grouping, schema, series)
|
||||
}
|
||||
|
||||
/// `topk`/`bottomk` with `k` evaluated independently at each grid point.
|
||||
pub fn topk_at(
|
||||
k_at: impl Fn(usize) -> Result<Option<usize>>,
|
||||
top: bool,
|
||||
grouping: &Grouping,
|
||||
schema: &SeriesSchema,
|
||||
series: &[Series],
|
||||
) -> Result<Vec<Series>> {
|
||||
single_field(schema, "topk/bottomk")?;
|
||||
// Validates the series against the schema and its grid; the walk below visits the carried
|
||||
// positions, not the whole grid.
|
||||
grid_points(schema, series)?;
|
||||
let Some(k) = k_param(k, series.len())? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let mut order: Vec<usize> = (0..series.len()).collect();
|
||||
order.sort_by(|&a, &b| compare_labels(&series[a].metric, &series[b].metric));
|
||||
let mut groups: HashMap<GroupKey, Vec<usize>> = HashMap::new();
|
||||
@@ -112,6 +121,9 @@ pub fn topk(
|
||||
}
|
||||
// Members are in label order, so a stable sort hands ties to the earlier label set.
|
||||
candidates.sort_by(|a, b| rank(a.2, b.2, top));
|
||||
let Some(k) = k_at(point)? else {
|
||||
continue;
|
||||
};
|
||||
for &(member, row, _) in candidates.iter().take(k) {
|
||||
selected[member].push((point, row));
|
||||
}
|
||||
@@ -525,7 +537,7 @@ fn rank(a: f64, b: f64, top: bool) -> Ordering {
|
||||
/// The `k` of `topk`, `bottomk` and `limitk` as Prometheus reads it: nothing for `k` below 1, an
|
||||
/// error for NaN or a value out of int64, otherwise `k` truncated and capped at the number of
|
||||
/// input series.
|
||||
fn k_param(k: f64, series: usize) -> Result<Option<usize>> {
|
||||
pub(crate) fn k_param(k: f64, series: usize) -> Result<Option<usize>> {
|
||||
if k < 1.0 {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -541,6 +553,44 @@ fn k_param(k: f64, series: usize) -> Result<Option<usize>> {
|
||||
Ok(Some(k.trunc().min(series as f64) as usize))
|
||||
}
|
||||
|
||||
/// Validates the complete parameter series for Prometheus topk/bottomk/limitk semantics.
|
||||
///
|
||||
/// The values are read where the node holds them, in one pass. The caller keeps both the series the
|
||||
/// parameter arrived as and the values it was read into, so a second copy of the present ones would
|
||||
/// be memory neither of them is charged for.
|
||||
pub(crate) fn validate_topk_parameters(values: &[Option<f64>]) -> Result<()> {
|
||||
// A position the series decided nothing at is no parameter value, so only the present ones are
|
||||
// read. NaN is tracked rather than folded into the bounds: it is a value Prometheus refuses
|
||||
// outright, not the largest one, and it is refused even where every value is below 1.
|
||||
let (mut max, mut min, mut nan) = (f64::NEG_INFINITY, f64::INFINITY, false);
|
||||
for value in values.iter().flatten() {
|
||||
if value.is_nan() {
|
||||
nan = true;
|
||||
} else {
|
||||
max = max.max(*value);
|
||||
min = min.min(*value);
|
||||
}
|
||||
}
|
||||
if nan {
|
||||
return Err(DataFusionError::Execution("Parameter value is NaN".into()));
|
||||
}
|
||||
// A `k` below 1 selects nothing, so a value that would not fit an int64 is never read.
|
||||
if max < 1.0 {
|
||||
return Ok(());
|
||||
}
|
||||
if min <= -9_223_372_036_854_775_808.0 {
|
||||
return Err(DataFusionError::Execution(format!(
|
||||
"Scalar value {min} underflows int64"
|
||||
)));
|
||||
}
|
||||
if max >= 9_223_372_036_854_774_784.0 {
|
||||
return Err(DataFusionError::Execution(format!(
|
||||
"Scalar value {max} overflows int64"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Label pairs identifying a group; empty values are absent, as in Prometheus.
|
||||
type GroupKey = Vec<(Arc<str>, Arc<str>)>;
|
||||
|
||||
|
||||
@@ -325,6 +325,12 @@ fn data_free_value_field(stmt: &EvalStmt) -> Arc<str> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// An aggregate's parameter is an expression since `topk` and friends take one, and
|
||||
// the grouping labels it binds are names the value field must not collide with:
|
||||
// an aggregate refuses to group by the field it folds.
|
||||
if let Some(param) = &aggregate.param {
|
||||
visit(param, targets);
|
||||
}
|
||||
visit(&aggregate.expr, targets);
|
||||
}
|
||||
PromExpr::Unary(unary) => visit(&unary.expr, targets),
|
||||
@@ -463,7 +469,8 @@ fn validate_inputs(root: &NativeNode, inputs: &[CompiledInput]) -> Result<()> {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::VectorOf { input } => check(input, inputs, next),
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => check(input, inputs, next),
|
||||
NativeExpr::Binary { left, right, .. }
|
||||
| NativeExpr::Set { left, right, .. }
|
||||
| NativeExpr::Info {
|
||||
@@ -479,6 +486,17 @@ fn validate_inputs(root: &NativeNode, inputs: &[CompiledInput]) -> Result<()> {
|
||||
.iter()
|
||||
.all(|parameter| check(parameter, inputs, next))
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => check(input, inputs, next) && check(parameter, inputs, next),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
check(input, inputs, next)
|
||||
&& parameters
|
||||
.iter()
|
||||
.all(|parameter| check(parameter, inputs, next))
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut next = 0;
|
||||
@@ -926,6 +944,23 @@ fn vector_aggregate(context: &Context, aggregate: &AggregateExpr) -> Result<Nati
|
||||
))
|
||||
})?;
|
||||
let group = grouping(modifier.as_ref());
|
||||
if matches!(op.id(), token::T_TOPK | token::T_BOTTOMK) && constant_value(param).is_none() {
|
||||
let input = compile_expr(context, expr)?;
|
||||
let output = input.output.clone();
|
||||
let parameter = compile_expr(context, param)?;
|
||||
return Ok(NativeNode {
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: op.id() == token::T_TOPK,
|
||||
group,
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
},
|
||||
output,
|
||||
});
|
||||
}
|
||||
let bound = match op.id() {
|
||||
token::T_TOPK | token::T_BOTTOMK => BoundVectorOp::TopK {
|
||||
k: constant_argument(&name, param, 0)?,
|
||||
@@ -1287,6 +1322,9 @@ fn compile_call(context: &Context, call: &Call) -> Result<NativeNode> {
|
||||
if let Some(signature) = range_signature(func.name) {
|
||||
return window_call(context, func.name, signature, &args.args);
|
||||
}
|
||||
if let Some(node) = parameterized_histogram_call(context, func.name, &args.args)? {
|
||||
return Ok(node);
|
||||
}
|
||||
if let Some((op, vector)) = vector_op(func.name, &args.args)? {
|
||||
return vector_call(context, func.name, op, vector);
|
||||
}
|
||||
@@ -1342,6 +1380,24 @@ fn scalar_call(
|
||||
&grid(),
|
||||
&context.value_field,
|
||||
),
|
||||
("scalar", [vector]) => {
|
||||
if vector.value_type() != ValueType::Vector {
|
||||
return Err(not_implemented("scalar() requires a vector argument"));
|
||||
}
|
||||
let input = compile_expr(context, vector)?;
|
||||
let NativeType::Samples(schema) = &input.output else {
|
||||
return Err(not_implemented("scalar() requires an evaluated vector"));
|
||||
};
|
||||
let Some(input_grid) = schema.eval_grid.clone() else {
|
||||
return Err(not_implemented("scalar() requires an evaluated vector"));
|
||||
};
|
||||
NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(input),
|
||||
},
|
||||
output: NativeType::Scalar(input_grid),
|
||||
}
|
||||
}
|
||||
(name, []) => match math_signature(name) {
|
||||
Some(MathSignature::Unary(MathOp::Date(date))) => {
|
||||
vector_of(eval_time(Some(date)), &grid(), &context.value_field)
|
||||
@@ -1526,6 +1582,61 @@ fn vector_call(
|
||||
})
|
||||
}
|
||||
|
||||
/// Compiles a classic histogram call with at least one varying scalar argument.
|
||||
fn parameterized_histogram_call(
|
||||
context: &Context,
|
||||
name: &str,
|
||||
arguments: &[Box<PromExpr>],
|
||||
) -> Result<Option<NativeNode>> {
|
||||
if !matches!(name, "histogram_quantile" | "histogram_fraction") {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some((vector, args)) = arguments.split_last() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let expected = if name == "histogram_quantile" { 1 } else { 2 };
|
||||
if args.len() != expected || args.iter().all(|arg| constant_value(arg).is_some()) {
|
||||
return Ok(None);
|
||||
}
|
||||
if vector.value_type() != ValueType::Vector {
|
||||
return Err(not_implemented(&format!(
|
||||
"a call to `{name}` whose vector argument is `{vector}` is not a vector"
|
||||
)));
|
||||
}
|
||||
let function = if name == "histogram_quantile" {
|
||||
HistogramFunction::Quantile(
|
||||
args.first()
|
||||
.and_then(|arg| constant_value(arg))
|
||||
.unwrap_or(0.0),
|
||||
)
|
||||
} else {
|
||||
HistogramFunction::Fraction {
|
||||
lower: args
|
||||
.first()
|
||||
.and_then(|arg| constant_value(arg))
|
||||
.unwrap_or(0.0),
|
||||
upper: args
|
||||
.get(1)
|
||||
.and_then(|arg| constant_value(arg))
|
||||
.unwrap_or(0.0),
|
||||
}
|
||||
};
|
||||
let input = compile_expr(context, vector)?;
|
||||
let output = input.output.clone();
|
||||
let parameters = args
|
||||
.iter()
|
||||
.map(|arg| compile_expr(context, arg))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Ok(Some(NativeNode {
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(function),
|
||||
parameters,
|
||||
},
|
||||
output,
|
||||
}))
|
||||
}
|
||||
|
||||
/// The vector operation a sort, `absent` or a classic histogram function binds, and the vector it reads,
|
||||
/// or `None` when `name` is none of them.
|
||||
fn vector_op<'a>(
|
||||
|
||||
@@ -289,6 +289,23 @@ fn a_long_statement_grid_is_deferred() {
|
||||
assert_eq!(grid_of(tree.root()), (0, 1, 86_400_001));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_topk_keeps_the_constant_vector_plan() {
|
||||
let tree = compile(
|
||||
&eval_stmt("topk(3, x)"),
|
||||
raw_schema(),
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
tree.root().expr,
|
||||
NativeExpr::Vector {
|
||||
op: promql::native::plan::BoundVectorOp::TopK { k: 3.0, .. },
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
/// `rate(x[5m])` becomes a window over the raw input, on the statement's grid.
|
||||
#[test]
|
||||
fn a_rate_over_a_selector_becomes_a_window_over_raw_input() {
|
||||
@@ -2247,6 +2264,13 @@ fn a_math_function_with_a_varying_parameter_binds_its_parameters_as_scalars() {
|
||||
NativeExpr::EvalTime { date: None }
|
||||
));
|
||||
|
||||
let scalar_tree =
|
||||
compile_statement(&eval_stmt("topk(scalar(vector(time())), vector(time()))")).unwrap();
|
||||
let NativeExpr::ParameterizedTopK { parameter, .. } = &scalar_tree.root().expr else {
|
||||
panic!("expected parameterized topk")
|
||||
};
|
||||
assert!(matches!(parameter.expr, NativeExpr::ScalarOf { .. }));
|
||||
|
||||
let tree = compiled("round(x, 2 * 5)").tree;
|
||||
assert!(matches!(
|
||||
map_op(tree.root()),
|
||||
@@ -2860,20 +2884,30 @@ fn the_aggregates_that_are_not_a_fold_bind_a_vector_operation() {
|
||||
assert_eq!(root, format!(" root Vector {op}"), "`{query}`");
|
||||
}
|
||||
|
||||
let error = compile(
|
||||
let compiled = compile_selectors(
|
||||
&eval_stmt("topk(scalar(x), x)"),
|
||||
&|_| Ok(raw_schema()),
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.expect("scalar(selector) is now compiled as a dynamic k");
|
||||
let NativeExpr::ParameterizedTopK { parameter, .. } = &compiled.tree.root().expr else {
|
||||
panic!("expected parameterized topk")
|
||||
};
|
||||
assert!(matches!(parameter.expr, NativeExpr::ScalarOf { .. }));
|
||||
|
||||
// `limitk` has no policy for a `k` that changes per point, so it keeps the constant rejection
|
||||
// and its message is the one that quotes the shapes this compiler accepts.
|
||||
let error = compile(
|
||||
&eval_stmt("limitk(scalar(x), x)"),
|
||||
raw_schema(),
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.expect_err("a run-time k compiled");
|
||||
assert!(
|
||||
matches!(error, DataFusionError::NotImplemented(_)),
|
||||
"{error}"
|
||||
);
|
||||
let message = error.to_string();
|
||||
assert!(message.contains("first argument"), "{message}");
|
||||
// The rejection quotes the accepted subset, so that quote has to name every aggregator this
|
||||
// compiler takes: one stale word there makes every rejection lie about what it binds.
|
||||
// That quote is the only description a reader of a rejection gets, so it has to name every
|
||||
// aggregator this compiler takes: one stale word there makes every rejection lie about what it
|
||||
// binds.
|
||||
for aggregator in [
|
||||
"sum",
|
||||
"avg",
|
||||
@@ -4283,6 +4317,7 @@ fn whole_vector_functions_bind_a_vector_operation() {
|
||||
"histogram_quantile(1 - 0.1, rate(x[5m]))",
|
||||
"histogram_quantile 0.9",
|
||||
),
|
||||
("histogram_quantile(0.9, x)", "histogram_quantile 0.9"),
|
||||
("histogram_fraction(0, 0.5, x)", "histogram_fraction 0 0.5"),
|
||||
("absent(rate(x[5m]))", "absent {}"),
|
||||
];
|
||||
@@ -5142,7 +5177,8 @@ fn validate_inputs_rejects_misbound_occurrences_and_accepts_scalar_wrappers() {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::VectorOf { input } => set_input_source(input, source),
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => set_input_source(input, source),
|
||||
NativeExpr::Binary { left, right, .. }
|
||||
| NativeExpr::Set { left, right, .. }
|
||||
| NativeExpr::Info {
|
||||
@@ -5153,6 +5189,20 @@ fn validate_inputs_rejects_misbound_occurrences_and_accepts_scalar_wrappers() {
|
||||
set_input_source(left, source);
|
||||
set_input_source(right, source);
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
set_input_source(input, source);
|
||||
set_input_source(parameter, source);
|
||||
}
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
set_input_source(input, source);
|
||||
for parameter in parameters {
|
||||
set_input_source(parameter, source);
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
|
||||
@@ -1550,12 +1550,19 @@ fn node_children(expr: &NativeExpr) -> Vec<&NativeNode> {
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => vec![input],
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::ScalarOf { input } => vec![input],
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => std::iter::once(input.as_ref())
|
||||
.chain(parameters.iter())
|
||||
.collect(),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => vec![input, parameter],
|
||||
NativeExpr::Binary { left, right, .. } | NativeExpr::Set { left, right, .. } => {
|
||||
vec![left, right]
|
||||
}
|
||||
@@ -1594,6 +1601,21 @@ fn with_node_children(
|
||||
op: *op,
|
||||
parameters: (0..parameters.len()).map(|_| next()).collect(),
|
||||
},
|
||||
NativeExpr::ScalarOf { .. } => NativeExpr::ScalarOf {
|
||||
input: Box::new(next()),
|
||||
},
|
||||
NativeExpr::ParameterizedTopK { op, .. } => NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(next()),
|
||||
op: op.clone(),
|
||||
parameter: Box::new(next()),
|
||||
},
|
||||
NativeExpr::ParameterizedHistogram { op, parameters, .. } => {
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(next()),
|
||||
op: op.clone(),
|
||||
parameters: (0..parameters.len()).map(|_| next()).collect(),
|
||||
}
|
||||
}
|
||||
NativeExpr::Timestamp { .. } => NativeExpr::Timestamp {
|
||||
input: Box::new(next()),
|
||||
},
|
||||
@@ -2694,10 +2716,7 @@ fn aggregate_reads_histogram(node: &NativeNode, input_kind: FieldKind) -> bool {
|
||||
!matches!(input.output_field_kind(input_kind), Some(FieldKind::Float))
|
||||
|| aggregate_reads_histogram(input, input_kind)
|
||||
}
|
||||
NativeExpr::Input(_)
|
||||
| NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::VectorOf { .. } => false,
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => false,
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Window { input, .. }
|
||||
@@ -2712,6 +2731,27 @@ fn aggregate_reads_histogram(node: &NativeNode, input_kind: FieldKind) -> bool {
|
||||
.iter()
|
||||
.any(|node| aggregate_reads_histogram(node, input_kind))
|
||||
}
|
||||
// `vector()` answers one float series, but the walk asks about aggregates anywhere in
|
||||
// the tree: `vector(scalar(sum(h)))` still folds histogram input. `scalar()` also reads
|
||||
// its input, and both wrappers can hide an aggregate that must be refused.
|
||||
NativeExpr::VectorOf { input } | NativeExpr::ScalarOf { input } => {
|
||||
aggregate_reads_histogram(input, input_kind)
|
||||
}
|
||||
// The dynamic nodes read their parameters beside their input, so every child matters.
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
aggregate_reads_histogram(input, input_kind)
|
||||
|| aggregate_reads_histogram(parameter, input_kind)
|
||||
}
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
aggregate_reads_histogram(input, input_kind)
|
||||
|| parameters
|
||||
.iter()
|
||||
.any(|node| aggregate_reads_histogram(node, input_kind))
|
||||
}
|
||||
NativeExpr::Binary { left, right, .. } | NativeExpr::Set { left, right, .. } => {
|
||||
aggregate_reads_histogram(left, input_kind)
|
||||
|| aggregate_reads_histogram(right, input_kind)
|
||||
@@ -2872,6 +2912,20 @@ fn native_output_tags(node: &NativeNode, source_tags: &[Vec<String>]) -> Option<
|
||||
NativeExpr::ParameterizedMap { input, .. } => {
|
||||
Some(without_metric_name(native_output_tags(input, source_tags)?))
|
||||
}
|
||||
// `scalar()` reports one value and no labels, like the scalar it produces.
|
||||
NativeExpr::ScalarOf { .. } => Some(Vec::new()),
|
||||
// Ranking keeps each selected series' own labels, and its parameter is a scalar.
|
||||
NativeExpr::ParameterizedTopK { input, .. } => native_output_tags(input, source_tags),
|
||||
// A parameterized histogram function drops the bucket bound and the metadata labels, as its
|
||||
// bound counterpart does; its parameters are scalars and carry no labels.
|
||||
NativeExpr::ParameterizedHistogram { input, .. } => native_output_tags(input, source_tags)
|
||||
.map(|tags| {
|
||||
tags.into_iter()
|
||||
.filter(|tag| {
|
||||
tag != "le" && tag != "__name__" && tag != "__type__" && tag != "__unit__"
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
NativeExpr::Vector { input, op } => match op {
|
||||
BoundVectorOp::TopK { .. }
|
||||
| BoundVectorOp::LimitK { .. }
|
||||
@@ -3059,19 +3113,27 @@ fn data_free(node: &NativeNode) -> bool {
|
||||
/// split declines `LimitK` for the same reason, and only a multi-input node repeats that refusal.
|
||||
fn depends_on_input_order(node: &NativeNode) -> bool {
|
||||
match &node.expr {
|
||||
NativeExpr::Input(_)
|
||||
| NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::VectorOf { .. } => false,
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => false,
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. } => depends_on_input_order(input),
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
// `vector()` and `scalar()` answer one series or one value from their input, and since
|
||||
// `scalar()` exists their input can hold an order-dependent selection:
|
||||
// `vector(scalar(limitk(1, a)))` keeps whichever `a` series the gather handed over first.
|
||||
| NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input } => depends_on_input_order(input),
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => depends_on_input_order(input) || parameters.iter().any(depends_on_input_order),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => depends_on_input_order(input) || depends_on_input_order(parameter),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => depends_on_input_order(input) || parameters.iter().any(depends_on_input_order),
|
||||
NativeExpr::Vector { input, op } => {
|
||||
matches!(op, BoundVectorOp::LimitK { .. }) || depends_on_input_order(input)
|
||||
}
|
||||
@@ -3105,6 +3167,9 @@ fn groups_by_metric_name(node: &NativeNode) -> bool {
|
||||
| NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::VectorOf { .. } => false,
|
||||
// `scalar()` answers one value with no labels at all, so a grouping by the metric name
|
||||
// below it cannot leave two groups under one label set: the walk stops there.
|
||||
NativeExpr::ScalarOf { .. } => false,
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Window { input, .. }
|
||||
@@ -3116,6 +3181,12 @@ fn groups_by_metric_name(node: &NativeNode) -> bool {
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => groups_by_metric_name(input) || parameters.iter().any(groups_by_metric_name),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => groups_by_metric_name(input) || groups_by_metric_name(parameter),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => groups_by_metric_name(input) || parameters.iter().any(groups_by_metric_name),
|
||||
NativeExpr::Vector { input, op } => {
|
||||
matches!(op, BoundVectorOp::CountValues { group: Grouping::By(names), .. }
|
||||
if names.iter().any(|name| name.as_ref() == METRIC_NAME))
|
||||
@@ -3163,11 +3234,15 @@ pub(crate) fn pointwise_operand(node: &NativeNode) -> bool {
|
||||
}
|
||||
}
|
||||
// An aggregate folds a group, a vector operation reads the whole vector, and a bare input or
|
||||
// scalar is not a per-series evaluation.
|
||||
// scalar is not a per-series evaluation. A parameterized topk or histogram reads the whole
|
||||
// vector, and `scalar()` answers from it.
|
||||
NativeExpr::Input(_)
|
||||
| NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::VectorOf { .. }
|
||||
| NativeExpr::ScalarOf { .. }
|
||||
| NativeExpr::ParameterizedTopK { .. }
|
||||
| NativeExpr::ParameterizedHistogram { .. }
|
||||
| NativeExpr::Aggregate { .. }
|
||||
| NativeExpr::Vector { .. }
|
||||
| NativeExpr::Set { .. }
|
||||
@@ -3237,6 +3312,20 @@ fn collect_selectors<'a>(
|
||||
// Literals contribute no selector; the compiler decides which scalar forms can be bound.
|
||||
PromExpr::NumberLiteral(_) => return true,
|
||||
PromExpr::Unary(unary) => return collect_selectors(stmt, &unary.expr, out),
|
||||
// `histogram_quantile` and `histogram_fraction` take their vector last, and the compiler
|
||||
// compiles that vector before the arguments in front of it, so the walk reads it first: the
|
||||
// scan prepared at a position has to be the one the compiler binds there.
|
||||
PromExpr::Call(call)
|
||||
if matches!(call.func.name, "histogram_quantile" | "histogram_fraction") =>
|
||||
{
|
||||
let Some((vector, parameters)) = call.args.args.split_last() else {
|
||||
return false;
|
||||
};
|
||||
return collect_selectors(stmt, vector, out)
|
||||
&& parameters
|
||||
.iter()
|
||||
.all(|parameter| collect_selectors(stmt, parameter, out));
|
||||
}
|
||||
PromExpr::Call(call) => {
|
||||
if matches!(call.func.name, "label_replace" | "label_join") {
|
||||
let arity_valid = if call.func.name == "label_replace" {
|
||||
@@ -3254,7 +3343,18 @@ fn collect_selectors<'a>(
|
||||
}
|
||||
}
|
||||
}
|
||||
PromExpr::Aggregate(aggregate) => return collect_selectors(stmt, &aggregate.expr, out),
|
||||
// The compiler compiles an aggregate's input and then its parameter (`topk`/`bottomk` take
|
||||
// an expression where the others take a number or a label), so a parameter that reads a
|
||||
// selector is a second source bound after the input. Walking both here is what keeps the
|
||||
// prepared scan and the bound input at the same position; a literal or a grouping-only
|
||||
// parameter contributes no selector, so the walk is a no-op for it.
|
||||
PromExpr::Aggregate(aggregate) => {
|
||||
return collect_selectors(stmt, &aggregate.expr, out)
|
||||
&& match &aggregate.param {
|
||||
Some(param) => collect_selectors(stmt, param, out),
|
||||
None => true,
|
||||
};
|
||||
}
|
||||
PromExpr::Subquery(_) => return false,
|
||||
_ => {}
|
||||
}
|
||||
@@ -3314,11 +3414,21 @@ fn calls_rate_or_increase(expr: &PromExpr) -> bool {
|
||||
|| call.args.args.iter().any(|arg| calls_rate_or_increase(arg))
|
||||
}
|
||||
PromExpr::Paren(ParenExpr { expr }) => calls_rate_or_increase(expr),
|
||||
PromExpr::Subquery(subquery) => calls_rate_or_increase(&subquery.expr),
|
||||
PromExpr::Binary(binary) => {
|
||||
calls_rate_or_increase(&binary.lhs) || calls_rate_or_increase(&binary.rhs)
|
||||
}
|
||||
PromExpr::Unary(unary) => calls_rate_or_increase(&unary.expr),
|
||||
PromExpr::Aggregate(aggregate) => calls_rate_or_increase(&aggregate.expr),
|
||||
// An aggregate's parameter is an expression, so `topk(scalar(sum(rate(x[5m]))), y)` reaches
|
||||
// `rate` only through it. The gate is about the whole statement, and a delta-temporality
|
||||
// table answers an increment there as it does anywhere else.
|
||||
PromExpr::Aggregate(aggregate) => {
|
||||
calls_rate_or_increase(&aggregate.expr)
|
||||
|| aggregate
|
||||
.param
|
||||
.as_ref()
|
||||
.is_some_and(|param| calls_rate_or_increase(param))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -3337,11 +3447,28 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
ArrowDataType, ArrowTimeUnit, FieldKind, METRIC_NAME, NATIVE_PROMQL_EXTENSION_KEY,
|
||||
NativeType, collect_selectors, cut_segments, fused_output_schema, fused_output_tags,
|
||||
hybrid_binary_admitted, independent_scan_pair, native_output_tags, native_promql_requested,
|
||||
native_raw_plan_bounds_region_reads, scan_time_filter,
|
||||
NativeType, aggregate_reads_histogram, calls_rate_or_increase, collect_selectors,
|
||||
cut_segments, fused_output_schema, fused_output_tags, hybrid_binary_admitted,
|
||||
independent_scan_pair, native_output_tags, native_promql_requested,
|
||||
native_raw_plan_bounds_region_reads, node_children, scan_time_filter,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn delta_temporality_guard_finds_rate_and_increase_inside_subqueries() {
|
||||
for query in [
|
||||
"rate(x[5m])",
|
||||
"increase(x[5m])",
|
||||
"avg_over_time(rate(x[5m])[10m:1m])",
|
||||
"avg_over_time(increase(x[5m])[10m:1m])",
|
||||
] {
|
||||
let expr = promql_parser::parser::parse(query).unwrap();
|
||||
assert!(calls_rate_or_increase(&expr), "{query}");
|
||||
}
|
||||
let query = "avg_over_time(abs(x)[10m:1m])";
|
||||
let expr = promql_parser::parser::parse(query).unwrap();
|
||||
assert!(!calls_rate_or_increase(&expr), "{query}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_raw_region_read_budget_needs_the_native_topology() {
|
||||
let schema = Arc::new(datafusion::arrow::datatypes::Schema::new(vec![
|
||||
@@ -3396,31 +3523,23 @@ mod tests {
|
||||
.tree
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_histogram_walk_descends_through_scalar_vector_wrappers() {
|
||||
let tree = compiled_test_tree("scalar(vector(scalar(sum(some_metric))))");
|
||||
assert_eq!(input_sources(tree.root()), vec![0]);
|
||||
assert!(aggregate_reads_histogram(tree.root(), FieldKind::Histogram));
|
||||
assert!(!aggregate_reads_histogram(tree.root(), FieldKind::Float));
|
||||
}
|
||||
|
||||
fn input_sources(root: &NativeNode) -> Vec<usize> {
|
||||
fn collect(node: &NativeNode, sources: &mut Vec<usize>) {
|
||||
match &node.expr {
|
||||
NativeExpr::Input(input) => sources.push(input.source_index),
|
||||
NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Timestamp { input } => collect(input, sources),
|
||||
NativeExpr::Binary { left, right, .. }
|
||||
| NativeExpr::Set { left, right, .. }
|
||||
| NativeExpr::Info {
|
||||
base: left,
|
||||
info: right,
|
||||
..
|
||||
} => {
|
||||
collect(left, sources);
|
||||
collect(right, sources);
|
||||
expr => {
|
||||
for child in node_children(expr) {
|
||||
collect(child, sources);
|
||||
}
|
||||
}
|
||||
NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::VectorOf { .. }
|
||||
| NativeExpr::ParameterizedMap { .. }
|
||||
| NativeExpr::Aggregate { .. }
|
||||
| NativeExpr::Vector { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3447,6 +3566,34 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The metric names the compiler binds selectors to, in the order it binds them.
|
||||
fn compiled_selector_order(query: &str) -> Vec<String> {
|
||||
let statement = promql_parser::parser::EvalStmt {
|
||||
expr: promql_parser::parser::parse(query).expect("the query parses"),
|
||||
start: UNIX_EPOCH,
|
||||
end: UNIX_EPOCH + Duration::from_secs(10),
|
||||
interval: Duration::from_secs(5),
|
||||
lookback_delta: Duration::from_secs(5),
|
||||
};
|
||||
let schema = Arc::new(SeriesSchema {
|
||||
fields: Arc::from([FieldSpec { name: "v0".into() }]),
|
||||
eval_grid: None,
|
||||
});
|
||||
let bound = std::cell::RefCell::new(Vec::new());
|
||||
crate::promql::native_compile::compile_selectors(
|
||||
&statement,
|
||||
&|requirement| {
|
||||
bound
|
||||
.borrow_mut()
|
||||
.push(requirement.metric.clone().unwrap_or_default());
|
||||
Ok(Arc::clone(&schema))
|
||||
},
|
||||
SeriesOrdering::KeyThenTime,
|
||||
)
|
||||
.expect("the compiler binds every selector of this query");
|
||||
bound.into_inner()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_walk_keeps_plain_selector_identity() {
|
||||
for (query, expected) in [
|
||||
@@ -3472,6 +3619,33 @@ mod tests {
|
||||
] {
|
||||
assert_eq!(collected_selectors(query), vec!["x"], "{query}");
|
||||
}
|
||||
// A parameter is an expression of its own, and the compiler compiles the input before it, so
|
||||
// a parameter that reads a selector is a second source bound after the input. The order here
|
||||
// is that binding order, which is the one the planner prepares its scans in.
|
||||
for (query, expected) in [
|
||||
("topk(scalar(a), b)", vec!["b", "a"]),
|
||||
("bottomk(scalar(a), b)", vec!["b", "a"]),
|
||||
// A literal parameter and a label parameter contribute no selector at all.
|
||||
("topk(3, b)", vec!["b"]),
|
||||
("count_values(\"x\", b)", vec!["b"]),
|
||||
(
|
||||
"histogram_quantile(scalar(a), sum by (le) (rate(h[5m])))",
|
||||
vec!["h", "a"],
|
||||
),
|
||||
(
|
||||
"histogram_fraction(scalar(a), scalar(b), sum by (le) (rate(h[5m])))",
|
||||
vec!["h", "a", "b"],
|
||||
),
|
||||
] {
|
||||
assert_eq!(collected_selectors(query), expected, "{query}");
|
||||
// The walk's contract is the compiler's binding order, so the two are compared rather
|
||||
// than kept in step by hand.
|
||||
assert_eq!(
|
||||
collected_selectors(query),
|
||||
compiled_selector_order(query),
|
||||
"{query}: the walk and the compiler must bind the same selectors in the same order"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The compiler retains both source indices for independent selector occurrences.
|
||||
@@ -3716,12 +3890,29 @@ mod tests {
|
||||
Some(vec!["host".to_string(), "job".to_string()]),
|
||||
native_output_tags(absent.root(), &sources)
|
||||
);
|
||||
// A per-point `k` keeps the input's labels too: it selects the same series the bound form
|
||||
// selects, so it reports the same set. Its parameter reads a source of its own, so this tree
|
||||
// binds two.
|
||||
assert_eq!(
|
||||
Some(vec!["hostname".to_string()]),
|
||||
native_output_tags(
|
||||
compiled_test_tree("topk(scalar(x), sum by (hostname) (rate(x[5m])))").root(),
|
||||
&[vec!["hostname".to_string()], vec!["hostname".to_string()]]
|
||||
),
|
||||
"a per-point k reports its input's labels"
|
||||
);
|
||||
let count_values =
|
||||
compiled_test_tree("count_values by (host) (\"sample\", sum by (host) (x))");
|
||||
assert_eq!(
|
||||
Some(vec!["host".to_string(), "sample".to_string()]),
|
||||
native_output_tags(count_values.root(), &sources)
|
||||
);
|
||||
// A destination already present in the grouping is reported once, not duplicated.
|
||||
let count_values = compiled_test_tree("count_values by (host) (\"host\", x)");
|
||||
assert_eq!(
|
||||
Some(vec!["host".to_string()]),
|
||||
native_output_tags(count_values.root(), &sources)
|
||||
);
|
||||
// Selecting some series keeps each one's own labels, whatever the grouping was.
|
||||
let topk = compiled_test_tree("topk(1, x)");
|
||||
assert_eq!(
|
||||
@@ -3803,6 +3994,13 @@ mod tests {
|
||||
Some(vec!["job".to_string(), "sample".to_string()]),
|
||||
native_output_tags(count_values.root(), &sources)
|
||||
);
|
||||
|
||||
let count_values =
|
||||
compiled_test_tree("count_values without (sample, instance) (\"sample\", x)");
|
||||
assert_eq!(
|
||||
Some(vec!["job".to_string()]),
|
||||
native_output_tags(count_values.root(), &sources)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -2429,8 +2429,13 @@ async fn native_fused_supports_constant_roots_and_selector_broadcasts() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A statement that reads no data plans without a scan. `scalar(v)` has no native node, so a
|
||||
/// statement that calls it keeps the existing plan.
|
||||
/// A statement that reads no data plans without a scan, and one that reads a scan through a scalar
|
||||
/// keeps the native route.
|
||||
///
|
||||
/// `scalar(v)` binds `NativeExpr::ScalarOf`, and a tree that holds it reads the whole vector: it is
|
||||
/// classified `SinglePartition`, so it is not cut per series and the statement plans natively. It
|
||||
/// must not be mistaken for a tree that keeps its source's matching key — that reading used to send
|
||||
/// it, and `topk`/`histogram_quantile` with it, back to the previous planner.
|
||||
#[tokio::test]
|
||||
async fn native_fused_plans_dynamic_scalars() {
|
||||
for query in ["time() - 60", "vector(time())", "hour()"] {
|
||||
@@ -2592,13 +2597,81 @@ async fn native_fused_plans_dynamic_scalars() {
|
||||
let (_, batches) = execute(plan, &build_query_engine_state()).await;
|
||||
assert!(!batches.is_empty(), "{query}");
|
||||
|
||||
for query in [
|
||||
"scalar(some_metric)",
|
||||
"some_metric > bool scalar(some_metric)",
|
||||
] {
|
||||
let plan = build_native_fused_plan(query, true).await;
|
||||
assert!(find_fused_series(&plan).is_none(), "{query}");
|
||||
// `scalar(m)` reads the whole vector: which series carries the one value at a point decides the
|
||||
// answer, so the tree runs where every series of `m` is gathered rather than per partition. That
|
||||
// still plans natively, as one fused node over the scan.
|
||||
let plan = build_native_fused_plan("scalar(some_metric)", true).await;
|
||||
let fused = find_fused_series(&plan)
|
||||
.unwrap_or_else(|| panic!("scalar() over a scan uses the native route: {plan}"));
|
||||
assert!(
|
||||
matches!(fused.tree().root().expr, NativeExpr::ScalarOf { .. }),
|
||||
"scalar() binds its own node: {plan}"
|
||||
);
|
||||
// The scalar side of a broadcast reads the whole vector, even when both sides read the same
|
||||
// table. Both selector occurrences bind their own source and the tree gathers before evaluation.
|
||||
let query = "some_metric > bool scalar(some_metric)";
|
||||
let plan = build_native_fused_plan(query, true).await;
|
||||
let fused = find_fused_series(&plan)
|
||||
.unwrap_or_else(|| panic!("same-table scalar broadcast uses the native route: {plan}"));
|
||||
assert_eq!(2, fused.inputs().len(), "{query}: {plan}");
|
||||
assert_eq!(
|
||||
vec!["some_metric".to_string(), "some_metric".to_string()],
|
||||
fused_input_metrics(&plan),
|
||||
"{query}: each selector occurrence binds its own source: {plan}"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
input_locality(fused.tree().root()),
|
||||
InputLocality::SinglePartition
|
||||
),
|
||||
"{query}: scalar() must see the whole vector: {plan}"
|
||||
);
|
||||
}
|
||||
|
||||
/// An aggregate's parameter is part of the tree a data-free statement names its value field from.
|
||||
///
|
||||
/// `topk(scalar(sum by (value) (vector(1))), vector(2))` groups by a label called `value`, so the
|
||||
/// value field has to step aside for it: an aggregate refuses to group by the field it folds, and
|
||||
/// the parameter is where that label is bound. Left out of the walk over the tree, the statement
|
||||
/// keeps the field name `value` and fails when the node is built for execution instead of answering
|
||||
/// `2`.
|
||||
#[tokio::test]
|
||||
async fn native_fused_plans_a_dynamic_topk_parameter_grouped_by_the_value_field() {
|
||||
let query = "topk(scalar(sum by (value) (vector(1))), vector(2))";
|
||||
let plan = build_native_fused_plan(query, true).await;
|
||||
{
|
||||
let fused = find_fused_series(&plan).expect("a data-free topk uses the native route");
|
||||
assert!(fused.children().is_empty(), "{query}");
|
||||
assert_eq!(
|
||||
vec!["native_value_0"],
|
||||
fused
|
||||
.tree()
|
||||
.root()
|
||||
.output
|
||||
.samples()
|
||||
.unwrap()
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| field.name.as_ref())
|
||||
.collect::<Vec<_>>(),
|
||||
"{query} native value field"
|
||||
);
|
||||
}
|
||||
let (_, batches) = execute(plan, &build_query_engine_state()).await;
|
||||
let mut answers = Vec::new();
|
||||
for batch in &batches {
|
||||
let values = batch
|
||||
.column(1)
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.unwrap();
|
||||
answers.extend((0..batch.num_rows()).map(|row| values.value(row)));
|
||||
}
|
||||
assert!(!answers.is_empty(), "{query} answered nothing");
|
||||
assert!(
|
||||
answers.iter().all(|value| *value == 2.0),
|
||||
"{query} answered {answers:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A map whose parameter reads no data runs per partition over its scan.
|
||||
@@ -3490,6 +3563,21 @@ async fn native_fused_binary_keeps_independent_scan_occurrences() {
|
||||
sources(left, out);
|
||||
sources(right, out);
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
sources(input, out);
|
||||
sources(parameter, out);
|
||||
}
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
sources(input, out);
|
||||
for parameter in parameters {
|
||||
sources(parameter, out);
|
||||
}
|
||||
}
|
||||
NativeExpr::ScalarOf { input } => sources(input, out),
|
||||
NativeExpr::Scalar(_)
|
||||
| NativeExpr::EvalTime { .. }
|
||||
| NativeExpr::VectorOf { .. }
|
||||
@@ -4123,15 +4211,25 @@ async fn native_fused_three_selector_trees_bind_every_occurrence() {
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Vector { input, .. }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::VectorOf { input } => child(input),
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
child(input);
|
||||
for parameter in parameters.iter() {
|
||||
child(parameter);
|
||||
}
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
child(input);
|
||||
child(parameter);
|
||||
}
|
||||
NativeExpr::Binary { left, right, .. } | NativeExpr::Set { left, right, .. } => {
|
||||
child(left);
|
||||
child(right);
|
||||
@@ -4260,6 +4358,10 @@ async fn native_fused_aggregate_splits_into_a_final_over_a_partial_producer() {
|
||||
// A map between the two: the constant is broadcast over a map above the fold, so the prefix
|
||||
// stays the selection and the suffix is `abs` over the merged value, then the multiply.
|
||||
"abs(sum(some_metric)) * 2",
|
||||
// A histogram whose parameter is a per-point constant is carried like the literal form: the
|
||||
// argument reads no series, so it means the same above the merged values and stays native.
|
||||
"histogram_quantile(time() / 150, sum by (le) (rate(some_metric[5m])))",
|
||||
"histogram_fraction(time() / 150, time() / 75 + 0.5, sum by (le) (rate(some_metric[5m])))",
|
||||
] {
|
||||
let plan = build_native_fused_plan(query, true).await;
|
||||
let display = plan.display_indent().to_string();
|
||||
@@ -4273,6 +4375,22 @@ async fn native_fused_aggregate_splits_into_a_final_over_a_partial_producer() {
|
||||
final_display.matches(" final").count(),
|
||||
"`{query}`: the merge must be a Final aggregate: {final_display}"
|
||||
);
|
||||
// A per-point parameter keeps the function native: the node the compiler bound for it sits
|
||||
// above the Final aggregate rather than the literal `Histogram` op the constant form binds.
|
||||
match query {
|
||||
"histogram_quantile(time() / 150, sum by (le) (rate(some_metric[5m])))" => assert!(
|
||||
final_display.contains("ParameterizedHistogram histogram_quantile"),
|
||||
"`{query}`: the split must carry the per-point parameter: {final_display}"
|
||||
),
|
||||
"histogram_fraction(time() / 150, time() / 75 + 0.5, sum by (le) (rate(some_metric[5m])))" =>
|
||||
{
|
||||
assert!(
|
||||
final_display.contains("ParameterizedHistogram histogram_fraction"),
|
||||
"`{query}`: the split must carry the per-point parameters: {final_display}"
|
||||
)
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let partial = partial_below(final_node).expect("the final node reads a partial producer");
|
||||
// The physical final must accept the producer's state and classify its own output fields.
|
||||
let input = Arc::new(datafusion::physical_plan::empty::EmptyExec::new(
|
||||
@@ -4347,6 +4465,12 @@ async fn native_fused_aggregate_splits_into_a_final_over_a_partial_producer() {
|
||||
"100 / sum(some_metric)" => &["field_0", "timestamp"],
|
||||
"-sum(some_metric)" => &["field_0", "timestamp"],
|
||||
"abs(sum(some_metric)) * 2" => &["field_0", "timestamp"],
|
||||
"histogram_quantile(time() / 150, sum by (le) (rate(some_metric[5m])))" => {
|
||||
&["field_0", "timestamp"]
|
||||
}
|
||||
"histogram_fraction(time() / 150, time() / 75 + 0.5, sum by (le) (rate(some_metric[5m])))" => {
|
||||
&["field_0", "timestamp"]
|
||||
}
|
||||
_ => panic!("missing exact native schema expectation for `{query}`"),
|
||||
};
|
||||
assert_eq!(
|
||||
@@ -4357,6 +4481,58 @@ async fn native_fused_aggregate_splits_into_a_final_over_a_partial_producer() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A whole-vector node over a bare scan is native, whichever form writes it: it reads every series of
|
||||
/// its source, so it is classified `SinglePartition` and never cut per series.
|
||||
///
|
||||
/// This is the regression for a `same_source` arm that passed the node through when its parameter was
|
||||
/// a per-point constant: that claimed the tree keeps its source's matching key, so the planner asked
|
||||
/// for `PartitionLocal`, refused its own `reads_across_series` answer and fell back to the previous
|
||||
/// planner for the per-point form only — while a tree inside `multi_input` would instead have ranked
|
||||
/// per partition and answered the union.
|
||||
#[tokio::test]
|
||||
async fn native_fused_whole_vector_nodes_over_a_bare_scan_stay_native() {
|
||||
for query in [
|
||||
// The literal ranking, which never had a `same_source` arm to go wrong.
|
||||
"topk(3, some_metric)",
|
||||
// The per-point ranking and the scalar, whose parameters are per-point constants. `time()`
|
||||
// is the bare constant a parameter-shaped `same_source` guard passes through; an arithmetic
|
||||
// expression over it never matched that guard, so both forms are pinned here.
|
||||
"topk(time() / 60 + 1, some_metric)",
|
||||
"topk(time(), some_metric)",
|
||||
"scalar(some_metric)",
|
||||
] {
|
||||
let plan = build_native_fused_plan(query, true).await;
|
||||
assert!(
|
||||
find_fused_series(&plan).is_some(),
|
||||
"`{query}` reads whole series at a point, so it must stay native: {plan}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// `topk` over a split aggregate needs a whole-vector boundary whether its `k` is bound or answered
|
||||
/// per point, so the two forms take the same route: native, and not split.
|
||||
#[tokio::test]
|
||||
async fn native_fused_topk_above_aggregate_declines_the_split_in_every_form() {
|
||||
for query in [
|
||||
// The literal `k` binds a constant `TopK` op, which `split` refuses above the cut.
|
||||
"topk(3, sum(some_metric))",
|
||||
// A per-point `k` ranks the same whole vector, so it refuses for the same reason rather
|
||||
// than becoming the one admitted form.
|
||||
"topk(time() / 1000, sum(some_metric))",
|
||||
] {
|
||||
let plan = build_native_fused_plan(query, true).await;
|
||||
let display = plan.display_indent().to_string();
|
||||
assert!(
|
||||
find_fused_series(&plan).is_some(),
|
||||
"`{query}` must stay native: {display}"
|
||||
);
|
||||
assert!(
|
||||
find_final_split(&plan).is_none(),
|
||||
"`{query}` must not split: {display}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A data-free vector stays in the gathered parent while the data-bearing prefix is cut.
|
||||
#[tokio::test]
|
||||
async fn native_fused_data_free_vector_does_not_block_segmentation() {
|
||||
@@ -4950,9 +5126,9 @@ async fn native_fused_histogram_quantile_binary_drops_le_from_operand_matching()
|
||||
}
|
||||
}
|
||||
|
||||
/// A shape neither the split nor the node can keep correct keeps the existing plan: a sibling a
|
||||
/// call computes has no native node at all, a group by a value field cannot be named by the state
|
||||
/// rows, and a kernel the compiler does not fold keeps it too.
|
||||
/// A shape neither the split nor the node can keep correct keeps the existing plan: a group by a
|
||||
/// value field cannot be named by the state rows, and a kernel the compiler does not fold keeps it
|
||||
/// too.
|
||||
///
|
||||
/// `sum(m) / m` and `m + sum(m)` are not on this list. The fold and the plain selector are two
|
||||
/// occurrences, so each reads its own source and the operation above them answers over both; the
|
||||
@@ -4968,9 +5144,6 @@ async fn native_fused_unsplittable_aggregate_falls_back() {
|
||||
for query in [
|
||||
"quantile(0.5, some_metric)",
|
||||
"sum(quantile(0.5, some_metric))",
|
||||
// A scalar a call computes is not a literal the split could keep: `scalar(...)` has no
|
||||
// native node, so the statement keeps the existing plan.
|
||||
"sum(some_metric) / scalar(some_metric)",
|
||||
"sum by (field_0) (some_metric)",
|
||||
] {
|
||||
let native = build_native_fused_plan(query, true).await;
|
||||
@@ -4993,14 +5166,15 @@ async fn native_fused_unsplittable_aggregate_falls_back() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A fold beside a plain selector is two occurrences, so each reads its own source and the fold and
|
||||
/// the match run above both. No partition needs to hold the whole of the input for that: the inputs
|
||||
/// are reconciled before the node evaluates them.
|
||||
/// A fold beside an independent selector is two occurrences, so each reads its own source and the
|
||||
/// operation runs above both. A scalar over the second occurrence requires the whole vector and
|
||||
/// therefore gathers the inputs before evaluation.
|
||||
#[tokio::test]
|
||||
async fn native_fused_aggregate_above_an_independent_selector_is_admitted() {
|
||||
for query in [
|
||||
"sum(some_metric) / some_metric",
|
||||
"some_metric + sum(some_metric)",
|
||||
"sum(some_metric) / scalar(some_metric)",
|
||||
] {
|
||||
let native = build_native_fused_plan(query, true).await;
|
||||
let fused = find_fused_series(&native).unwrap_or_else(|| panic!("`{query}`: {native}"));
|
||||
@@ -5010,6 +5184,15 @@ async fn native_fused_aggregate_above_an_independent_selector_is_admitted() {
|
||||
fused_input_metrics(&native),
|
||||
"`{query}`: each selector occurrence reads its own source: {native}"
|
||||
);
|
||||
if query.contains("scalar(") {
|
||||
assert!(
|
||||
matches!(
|
||||
input_locality(fused.tree().root()),
|
||||
InputLocality::SinglePartition
|
||||
),
|
||||
"`{query}`: scalar() must see the whole vector: {native}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
find_final_split(&native).is_none(),
|
||||
"`{query}`: a tree over two inputs is not a one-input split: {native}"
|
||||
@@ -12948,3 +13131,55 @@ async fn eval_times_before_epoch() {
|
||||
assert!(plan(too_early, too_early).await.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
/// A grouping by the metric name inside `scalar()` cannot reach the output: the scalar clears the
|
||||
/// labels, so the walk must stop there instead of refusing the statement.
|
||||
///
|
||||
/// Refusing it is not a quiet fallback - the existing planner takes both operands as vector inputs
|
||||
/// and reports `MultipleVector`, so a valid query fails outright.
|
||||
#[tokio::test]
|
||||
async fn native_fused_scalar_clears_metric_name_grouping() {
|
||||
let tables = [
|
||||
(DEFAULT_SCHEMA_NAME.to_string(), "a".to_string()),
|
||||
(DEFAULT_SCHEMA_NAME.to_string(), "b".to_string()),
|
||||
];
|
||||
let query = "clamp_min(b, scalar(sum by (__name__) (a)))";
|
||||
let provider = build_test_table_provider_with_tsid(&tables, 2, 1).await;
|
||||
let plan = build_native_fused_plan_with_provider(query, true, provider).await;
|
||||
let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}"));
|
||||
assert_eq!(fused.inputs().len(), 2, "{query}");
|
||||
}
|
||||
|
||||
/// Count-values output includes its generated label after grouping across logical tables.
|
||||
#[tokio::test]
|
||||
async fn native_fused_metric_engine_count_values_output_labels() {
|
||||
let tables = [
|
||||
(DEFAULT_SCHEMA_NAME.to_string(), "a".to_string()),
|
||||
(DEFAULT_SCHEMA_NAME.to_string(), "b".to_string()),
|
||||
];
|
||||
for (query, expected_tags) in [
|
||||
(r#"count_values("sample", a or b)"#, vec!["sample"]),
|
||||
(
|
||||
r#"count_values("sample", a or b) by (tag_0)"#,
|
||||
vec!["sample", "tag_0"],
|
||||
),
|
||||
(
|
||||
r#"count_values("sample", a or b) without (sample)"#,
|
||||
vec!["tag_0", "tag_1"],
|
||||
),
|
||||
] {
|
||||
let provider = build_test_table_provider_with_tsid(&tables, 2, 1).await;
|
||||
let plan = build_native_fused_plan_with_provider(query, true, provider).await;
|
||||
let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}"));
|
||||
assert_eq!(fused.inputs().len(), 2, "{query}");
|
||||
let mut expected = expected_tags
|
||||
.into_iter()
|
||||
.map(str::to_string)
|
||||
.collect::<Vec<_>>();
|
||||
expected.push("timestamp".to_string());
|
||||
// The value column keeps the compiled tree's field name rather than the expression text.
|
||||
expected.push("field_0".to_string());
|
||||
expected.sort();
|
||||
assert_eq!(sorted_column_names(&plan), expected, "{query}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2481,34 +2481,65 @@ async fn native_aggregate_complete_over_four_nonempty_mito_partitions() {
|
||||
)
|
||||
.await;
|
||||
|
||||
// The shapes the native compiler declines, each declined for its own reason, and each checked on
|
||||
// the plans that really ran: none may hold a fused producer or a frontend merge, and both routes
|
||||
// must answer what the fixture derives by hand.
|
||||
//
|
||||
// `scalar` reports the one value present at a point only when exactly one series is present at
|
||||
// it, so the five counter series make it NaN; a route that folded one partition at a time would
|
||||
// report the number of whichever partition held a single series, so the NaN also says the whole
|
||||
// vector was read at once. A subquery evaluates its own body on a grid of its own: it reads
|
||||
// across series and cannot be answered by a producer per partition, so it keeps the existing
|
||||
// plan. The subquery's window
|
||||
// spans the two inner points `EVAL_SECS - 1` and `EVAL_SECS`, at each of which the same
|
||||
// extrapolated rate answers 15 - both samples stay further from the window's ends than the 66s
|
||||
// threshold - so the maximum is 15 as well.
|
||||
let fallback_cases = [
|
||||
case(
|
||||
format!("scalar({COUNTER_TABLE})"),
|
||||
&[(&[], f64::NAN)],
|
||||
false,
|
||||
),
|
||||
case(
|
||||
format!("max_over_time((sum(rate({COUNTER_TABLE}[5m])))[2s:1s])"),
|
||||
&[(&[], 15.0)],
|
||||
false,
|
||||
),
|
||||
];
|
||||
for case in &fallback_cases {
|
||||
check_fallback_case(&instance, &observer, case).await;
|
||||
}
|
||||
// A shape the native compiler declines, checked on the plans that really ran: it may hold
|
||||
// neither a fused producer nor a frontend merge, and both routes must answer what the fixture
|
||||
// derives by hand. A subquery evaluates its own body on a grid of its own: it reads across
|
||||
// series and cannot be answered by a producer per partition, so it keeps the existing plan. The
|
||||
// subquery's window spans the two inner points `EVAL_SECS - 1` and `EVAL_SECS`, at each of
|
||||
// which the same extrapolated rate answers 15 - both samples stay further from the window's
|
||||
// ends than the 66s threshold - so the maximum is 15 as well.
|
||||
let fallback_case = case(
|
||||
format!("max_over_time((sum(rate({COUNTER_TABLE}[5m])))[2s:1s])"),
|
||||
&[(&[], 15.0)],
|
||||
false,
|
||||
);
|
||||
check_fallback_case(&instance, &observer, &fallback_case).await;
|
||||
|
||||
// `scalar` reads the whole vector and the native route answers it, so it is no longer one of
|
||||
// the declined shapes. `scalar` reports the one value present at a point only when exactly one
|
||||
// series is present at it, so the five counter series make it NaN; a route that folded one
|
||||
// partition at a time would report the value of whichever partition held a single series, so
|
||||
// the NaN also says the whole vector was read at once. The native run must therefore hold a
|
||||
// fused producer and no frontend merge, and answer the same rows the existing plan answers.
|
||||
let scalar_case = case(
|
||||
format!("scalar({COUNTER_TABLE})"),
|
||||
&[(&[], f64::NAN)],
|
||||
false,
|
||||
);
|
||||
let scalar_query = tql_eval(&scalar_case.promql);
|
||||
let (legacy, legacy_capture) =
|
||||
run_query(&instance, &scalar_case.promql, false, &observer).await;
|
||||
assert!(
|
||||
legacy_capture.fused.is_empty() && legacy_capture.finalize.is_empty(),
|
||||
"`{scalar_query}`: the legacy run must not plan a native node, but it planned {} fused \
|
||||
producers and {} frontend merges",
|
||||
legacy_capture.fused.len(),
|
||||
legacy_capture.finalize.len()
|
||||
);
|
||||
assert_rows(&legacy.rows, &scalar_case.expected, "legacy `scalar`");
|
||||
let mark = observer.mark();
|
||||
let (native, capture) = run_query(&instance, &scalar_case.promql, true, &observer).await;
|
||||
assert!(
|
||||
observer.mark() > mark,
|
||||
"`{scalar_query}`: the native run handed no plan to the observer, so nothing about the \
|
||||
plan that ran was observed"
|
||||
);
|
||||
assert!(
|
||||
!capture.fused.is_empty(),
|
||||
"`{scalar_query}`: `scalar` over a scan runs the native route, but the plans that ran held \
|
||||
no fused producer"
|
||||
);
|
||||
assert!(
|
||||
capture.finalize.is_empty(),
|
||||
"`{scalar_query}`: `scalar` needs no frontend merge, but {} ran",
|
||||
capture.finalize.len()
|
||||
);
|
||||
assert_rows(&native.rows, &scalar_case.expected, "native `scalar`");
|
||||
assert_rows(
|
||||
&native.rows,
|
||||
&legacy.rows,
|
||||
"native `scalar` against the existing plan",
|
||||
);
|
||||
}
|
||||
|
||||
/// Histogram functions above the cut merge Partial state before evaluating the complete buckets.
|
||||
|
||||
@@ -1849,6 +1849,23 @@ impl NativePlanObserver {
|
||||
fn fused(&self) -> Option<Arc<dyn datafusion::physical_plan::ExecutionPlan>> {
|
||||
self.0.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// The name of the node that answers the statement, when it is one of the native nodes.
|
||||
///
|
||||
/// [`Self::fused`] answers whether *some* fused node exists, and a split plan holds two: the
|
||||
/// partition-local producer below the state router and the merge above it. A test that means
|
||||
/// "the whole statement runs natively" therefore has to say which node answers it, not that one
|
||||
/// was captured. The search is breadth first, so the shallowest native node is the one reported.
|
||||
fn answering_node(&self) -> Option<String> {
|
||||
let mut queue = std::collections::VecDeque::from([self.root_plan()]);
|
||||
while let Some(node) = queue.pop_front() {
|
||||
if node.name() == "FusedSeriesExec" {
|
||||
return Some(node.name().to_string());
|
||||
}
|
||||
queue.extend(node.children().into_iter().cloned());
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Unique physical nodes reachable from the frontend and separately wrapped region trees.
|
||||
@@ -8327,36 +8344,44 @@ async fn native_promql_declines_a_selection_that_depends_on_arrival_order() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let query = "native_order_left and on(host) limitk(1, native_order_right)";
|
||||
let sql = format!("TQL EVAL (0, 0, '1s') {query}");
|
||||
for query in [
|
||||
"native_order_left and on(host) limitk(1, native_order_right)",
|
||||
// The same selection under a `vector(scalar(...))` wrapper. The wrapper answers one value
|
||||
// from its input, so the order-dependent selection is still in the tree: a walk that read
|
||||
// `vector()` as a leaf would admit this shape and answer `10` or `12` depending on which
|
||||
// partition the gather handed over first.
|
||||
"vector(scalar(limitk(1, native_order_left))) + on() native_order_right",
|
||||
] {
|
||||
let sql = format!("TQL EVAL (0, 0, '1s') {query}");
|
||||
|
||||
// The route declines, so the statement is planned by the one that was always its answer.
|
||||
observer.clear();
|
||||
let native = instance
|
||||
.do_query(&sql, native_query_context(true, 4))
|
||||
.await
|
||||
.remove(0);
|
||||
assert!(
|
||||
observer.fused().is_none(),
|
||||
"{query}: the route must decline a selection whose answer depends on arrival order"
|
||||
);
|
||||
let legacy = instance
|
||||
.do_query(&sql, native_query_context(false, 4))
|
||||
.await
|
||||
.remove(0);
|
||||
// Both routes report the same outcome: the decline leads to the plan this statement already had,
|
||||
// not to an answer the schedule could change.
|
||||
assert_eq!(
|
||||
native.is_err(),
|
||||
legacy.is_err(),
|
||||
"{query}: the route must not change whether the statement is answerable"
|
||||
);
|
||||
if let (Err(native_error), Err(legacy_error)) = (native, legacy) {
|
||||
assert_eq!(
|
||||
native_error.to_string(),
|
||||
legacy_error.to_string(),
|
||||
"{query}: the decline must lead to the plan this statement already had"
|
||||
// The route declines, so the statement is planned by the one that was always its answer.
|
||||
observer.clear();
|
||||
let native = instance
|
||||
.do_query(&sql, native_query_context(true, 4))
|
||||
.await
|
||||
.remove(0);
|
||||
assert!(
|
||||
observer.fused().is_none(),
|
||||
"{query}: the route must decline a selection whose answer depends on arrival order"
|
||||
);
|
||||
let legacy = instance
|
||||
.do_query(&sql, native_query_context(false, 4))
|
||||
.await
|
||||
.remove(0);
|
||||
// Both routes report the same outcome: the decline leads to the plan this statement already
|
||||
// had, not to an answer the schedule could change.
|
||||
assert_eq!(
|
||||
native.is_err(),
|
||||
legacy.is_err(),
|
||||
"{query}: the route must not change whether the statement is answerable"
|
||||
);
|
||||
if let (Err(native_error), Err(legacy_error)) = (native, legacy) {
|
||||
assert_eq!(
|
||||
native_error.to_string(),
|
||||
legacy_error.to_string(),
|
||||
"{query}: the decline must lead to the plan this statement already had"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9147,6 +9172,75 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() {
|
||||
);
|
||||
}
|
||||
|
||||
/// A same-table scalar broadcast gathers simultaneous series from distinct physical regions.
|
||||
/// `scalar(m)` is NaN for two series, so `m == bool scalar(m)` answers zero for both. Evaluating
|
||||
/// the scalar separately in each region would instead compare each series to itself and answer one.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn native_promql_same_table_partitioned_scalar_broadcast_gathers_whole_vector() {
|
||||
let (standalone, observer) = native_test_instance(GreptimeDbStandaloneBuilder::new(
|
||||
"native_promql_partitioned_scalar_broadcast",
|
||||
))
|
||||
.await;
|
||||
let instance = standalone.fe_instance().clone();
|
||||
execute_all(
|
||||
&instance,
|
||||
"CREATE TABLE native_partitioned_scalar (host STRING, ts TIMESTAMP TIME INDEX, \
|
||||
val DOUBLE, PRIMARY KEY(host)) PARTITION ON COLUMNS (host) (host < 'm', host >= 'm'); \
|
||||
INSERT INTO native_partitioned_scalar VALUES ('a', 5000, 2), ('z', 5000, 7)",
|
||||
QueryContext::arc(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let sql =
|
||||
"TQL EVAL (5, 5, '1s') native_partitioned_scalar == bool scalar(native_partitioned_scalar)";
|
||||
observer.clear();
|
||||
let rows = native_result_rows(native_tql_output(&instance, sql, true).await).await;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![(5_000, "a".to_string(), 0.0), (5_000, "z".to_string(), 0.0)],
|
||||
"the global scalar is NaN and equality is false for both series"
|
||||
);
|
||||
|
||||
let root = observer.root_plan();
|
||||
let fused = find_plan_node(&root, "FusedSeriesExec").unwrap_or_else(|| {
|
||||
panic!(
|
||||
"the frontend must answer natively: {}",
|
||||
display_plan_tree(&root)
|
||||
)
|
||||
});
|
||||
assert_eq!(
|
||||
observer.answering_node().as_deref(),
|
||||
Some("FusedSeriesExec")
|
||||
);
|
||||
assert_eq!(
|
||||
fused.children().len(),
|
||||
2,
|
||||
"one input per selector: {}",
|
||||
display_plan_tree(&root)
|
||||
);
|
||||
assert_eq!(
|
||||
fused.output_partitioning().partition_count(),
|
||||
1,
|
||||
"the whole vector is gathered: {}",
|
||||
display_plan_tree(&root)
|
||||
);
|
||||
let shape = native_plan_shape(&root);
|
||||
assert_eq!(
|
||||
shape.regions_per_scan,
|
||||
vec![2, 2],
|
||||
"each selector reads both declared host ranges: {}",
|
||||
display_plan_tree(&root)
|
||||
);
|
||||
// The disjoint declared ranges place `a` and `z` in different physical regions; both scans
|
||||
// select those regions. Compare the two NaN equalities with a literal false control as well.
|
||||
let literal = "TQL EVAL (5, 5, '1s') native_partitioned_scalar == bool 0";
|
||||
let legacy = native_result_rows(native_tql_output(&instance, literal, false).await).await;
|
||||
assert_eq!(
|
||||
rows, legacy,
|
||||
"a literal false comparison yields the same labelled rows"
|
||||
);
|
||||
}
|
||||
|
||||
/// The `ts` and first `Float64` column of a `TQL EVAL` answer, as values.
|
||||
///
|
||||
/// Answers are compared this way rather than as printed tables: the native route names a field
|
||||
@@ -10518,6 +10612,447 @@ async fn native_promql_real_query_q04_namespace_cpu_requests() {
|
||||
);
|
||||
}
|
||||
|
||||
/// A multi-field `scalar()` input declines the native route and keeps the existing plan.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn native_promql_scalar_multi_field_falls_back() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let (standalone, observer) = native_test_instance(GreptimeDbStandaloneBuilder::new(
|
||||
"native_promql_scalar_multi_field_fallback",
|
||||
))
|
||||
.await;
|
||||
let instance = standalone.fe_instance().clone();
|
||||
execute_all(
|
||||
&instance,
|
||||
"CREATE TABLE native_scalar_multi (host STRING, ts TIMESTAMP TIME INDEX, val DOUBLE, val2 DOUBLE, PRIMARY KEY(host)); \
|
||||
INSERT INTO native_scalar_multi (host, ts, val, val2) VALUES ('a', 0, 1, 10)",
|
||||
QueryContext::arc(),
|
||||
)
|
||||
.await;
|
||||
let sql = "TQL EVAL (0, 0, '1s') scalar(native_scalar_multi)";
|
||||
let error = instance
|
||||
.do_query(sql, native_query_context(true, 4))
|
||||
.await
|
||||
.remove(0)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
format!("{error:?}").contains("Multi fields calculation is not supported in scalar"),
|
||||
"unexpected multi-field scalar error: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
observer.fused().is_none(),
|
||||
"multi-field scalar() must fall back"
|
||||
);
|
||||
}
|
||||
|
||||
/// A dynamic classic histogram quantile matches each equivalent constant-phi evaluation.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn native_promql_dynamic_histogram_quantile() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let config = mito2::config::MitoConfig {
|
||||
experimental_series_scan_v2: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (standalone, observer) = native_test_instance(
|
||||
GreptimeDbStandaloneBuilder::new("native_dynamic_histogram_quantile")
|
||||
.with_mito_config(config),
|
||||
)
|
||||
.await;
|
||||
let instance = standalone.fe_instance().clone();
|
||||
execute_all(&instance, "CREATE TABLE native_dyn_hist (ts TIMESTAMP TIME INDEX, le STRING, v DOUBLE, PRIMARY KEY(le)); INSERT INTO native_dyn_hist (ts, le, v) VALUES (-60000, '1', 0), (0, '1', 5), (60000, '1', 10), (-60000, '2', 0), (0, '2', 9), (60000, '2', 18), (-60000, '+Inf', 0), (0, '+Inf', 10), (60000, '+Inf', 20)", QueryContext::arc()).await;
|
||||
let query = "sum by(le)(rate(native_dyn_hist[5m]))";
|
||||
let sql = format!("TQL EVAL (0, 60, '60s') histogram_quantile(time() / 150 + 0.5, {query})");
|
||||
observer.clear();
|
||||
let native = native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"dynamic histogram query did not run natively"
|
||||
);
|
||||
// The legacy planner requires literal histogram parameters, so compare each point with its constant-phi equivalent.
|
||||
let mut expected = Vec::new();
|
||||
for (t, phi) in [(0, 0.5), (60, 0.9)] {
|
||||
let literal = format!("TQL EVAL ({t}, {t}, '60s') histogram_quantile({phi}, {query})");
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
expected.append(&mut rows);
|
||||
}
|
||||
let mut actual = native.1;
|
||||
actual.sort_by_key(|row| row.0);
|
||||
expected.sort_by_key(|row| row.0);
|
||||
assert_eq!(actual, expected);
|
||||
|
||||
// A literal lower bound and a dynamic upper bound exercise both parameter paths together.
|
||||
let fraction_sql =
|
||||
format!("TQL EVAL (0, 60, '60s') histogram_fraction(0, time() / 150 + 0.5, {query})");
|
||||
observer.clear();
|
||||
let fraction_native =
|
||||
native_labelled_rows(native_tql_output(&instance, &fraction_sql, true).await).await;
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"dynamic histogram fraction did not run natively: {fraction_sql}"
|
||||
);
|
||||
let mut fraction_expected = Vec::new();
|
||||
for (t, upper) in [(0, 0.5), (60, 0.9)] {
|
||||
let literal = format!("TQL EVAL ({t}, {t}, '60s') histogram_fraction(0, {upper}, {query})");
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
fraction_expected.append(&mut rows);
|
||||
}
|
||||
let mut fraction_actual = fraction_native.1;
|
||||
fraction_actual.sort_by_key(|row| row.0);
|
||||
fraction_expected.sort_by_key(|row| row.0);
|
||||
assert_eq!(fraction_actual, fraction_expected, "{fraction_sql}");
|
||||
|
||||
let wrapped_query =
|
||||
"abs(histogram_quantile(time() / 150 + 0.5, sum by(le)(rate(native_dyn_hist[5m]))))";
|
||||
let wrapped_sql = format!("TQL EVAL (0, 60, '60s') {wrapped_query}");
|
||||
observer.clear();
|
||||
let wrapped_native =
|
||||
native_labelled_rows(native_tql_output(&instance, &wrapped_sql, true).await).await;
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"wrapped dynamic histogram query did not run natively: {wrapped_sql}"
|
||||
);
|
||||
let mut wrapped_expected = Vec::new();
|
||||
for (t, phi) in [(0, 0.5), (60, 0.9)] {
|
||||
let literal = format!("TQL EVAL ({t}, {t}, '60s') abs(histogram_quantile({phi}, {query}))");
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
wrapped_expected.append(&mut rows);
|
||||
}
|
||||
let mut wrapped_actual = wrapped_native.1;
|
||||
wrapped_actual.sort_by_key(|row| row.0);
|
||||
wrapped_expected.sort_by_key(|row| row.0);
|
||||
assert_eq!(wrapped_actual, wrapped_expected, "{wrapped_sql}");
|
||||
}
|
||||
|
||||
/// A dynamic `topk` parameter is evaluated at each TQL evaluation timestamp. `time() / 60 + 1`
|
||||
/// truncates to k = 1, 2, 3 at 0s, 60s, and 120s respectively.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn native_promql_dynamic_topk_matches_legacy() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let config = mito2::config::MitoConfig {
|
||||
experimental_series_scan_v2: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (standalone, observer) = native_test_instance(
|
||||
GreptimeDbStandaloneBuilder::new("native_promql_dynamic_topk_e2e").with_mito_config(config),
|
||||
)
|
||||
.await;
|
||||
let instance = standalone.fe_instance().clone();
|
||||
execute_all(
|
||||
&instance,
|
||||
r#"CREATE TABLE native_dyn_topk (host STRING, ts TIMESTAMP TIME INDEX, val DOUBLE, PRIMARY KEY(host));
|
||||
INSERT INTO native_dyn_topk (host, ts, val) VALUES
|
||||
('h0', -60000, 0), ('h0', 0, 1), ('h0', 60000, 2), ('h0', 120000, 3),
|
||||
('h1', -60000, 0), ('h1', 0, 2), ('h1', 60000, 4), ('h1', 120000, 6),
|
||||
('h2', -60000, 0), ('h2', 0, 3), ('h2', 60000, 6), ('h2', 120000, 9),
|
||||
('h3', -60000, 0), ('h3', 0, 4), ('h3', 60000, 8), ('h3', 120000, 12),
|
||||
('h4', -60000, 0), ('h4', 0, 5), ('h4', 60000, 10), ('h4', 120000, 15),
|
||||
('h5', -60000, 0), ('h5', 0, 6), ('h5', 60000, 12), ('h5', 120000, 18),
|
||||
('h6', -60000, 0), ('h6', 0, 7), ('h6', 60000, 14), ('h6', 120000, 21),
|
||||
('h7', -60000, 0), ('h7', 0, 8), ('h7', 60000, 16), ('h7', 120000, 24)"#,
|
||||
QueryContext::arc(),
|
||||
)
|
||||
.await;
|
||||
let query_body = "sum by(host)(rate(native_dyn_topk[5m]))";
|
||||
let sql = format!("TQL EVAL (0, 120, '60s') topk(time() / 60 + 1, {query_body})");
|
||||
observer.clear();
|
||||
let native = native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"dynamic topk did not run natively"
|
||||
);
|
||||
let counts = [0, 60_000, 120_000].map(|timestamp| {
|
||||
native
|
||||
.1
|
||||
.iter()
|
||||
.filter(|(ts, _, _)| *ts == timestamp)
|
||||
.count()
|
||||
});
|
||||
assert_eq!(counts, [1, 2, 3], "k = 1, 2, 3 at 0s, 60s, 120s");
|
||||
// The legacy planner requires a literal topk parameter, so compare each timestamp against
|
||||
// its equivalent literal-k query (the k values are computed above from `time() / 60 + 1`).
|
||||
let mut legacy_rows = Vec::new();
|
||||
for (timestamp_seconds, k) in [(0, 1), (60, 2), (120, 3)] {
|
||||
let legacy_sql = format!(
|
||||
"TQL EVAL ({timestamp_seconds}, {timestamp_seconds}, '60s') topk({k}, {query_body})"
|
||||
);
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &legacy_sql, false).await).await;
|
||||
legacy_rows.append(&mut rows);
|
||||
}
|
||||
let mut native_rows = native.1.clone();
|
||||
native_rows.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
legacy_rows.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(native_rows, legacy_rows);
|
||||
|
||||
for is_abs in [true, false] {
|
||||
let wrapped_query = if is_abs {
|
||||
format!("abs(topk(time() / 60 + 1, {query_body}))")
|
||||
} else {
|
||||
format!("topk(time() / 60 + 1, {query_body}) + 1")
|
||||
};
|
||||
let wrapped_sql = format!("TQL EVAL (0, 120, '60s') {wrapped_query}");
|
||||
observer.clear();
|
||||
let wrapped_native =
|
||||
native_labelled_rows(native_tql_output(&instance, &wrapped_sql, true).await).await;
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"wrapped dynamic topk did not run natively: {wrapped_sql}"
|
||||
);
|
||||
let mut wrapped_expected = Vec::new();
|
||||
for (timestamp_seconds, k) in [(0, 1), (60, 2), (120, 3)] {
|
||||
let wrapped_literal = if is_abs {
|
||||
format!("abs(topk({k}, {query_body}))")
|
||||
} else {
|
||||
format!("topk({k}, {query_body}) + 1")
|
||||
};
|
||||
let literal_sql = format!(
|
||||
"TQL EVAL ({timestamp_seconds}, {timestamp_seconds}, '60s') {wrapped_literal}"
|
||||
);
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal_sql, false).await).await;
|
||||
wrapped_expected.append(&mut rows);
|
||||
}
|
||||
let mut wrapped_actual = wrapped_native.1;
|
||||
wrapped_actual.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
wrapped_expected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(wrapped_actual, wrapped_expected, "{wrapped_sql}");
|
||||
}
|
||||
|
||||
let outer_sql =
|
||||
format!("TQL EVAL (0, 120, '60s') topk(1, topk(time() / 60 + 1, {query_body}))");
|
||||
observer.clear();
|
||||
let outer_native =
|
||||
native_labelled_rows(native_tql_output(&instance, &outer_sql, true).await).await;
|
||||
// `topk` above a split aggregate needs a whole-vector boundary, so no split answers this
|
||||
// statement: the one fused node that reads the series answers it whole.
|
||||
assert_eq!(
|
||||
observer.answering_node().as_deref(),
|
||||
Some("FusedSeriesExec"),
|
||||
"outer constant topk over a dynamic topk did not answer natively: {outer_sql}"
|
||||
);
|
||||
let mut outer_expected = Vec::new();
|
||||
for (timestamp_seconds, k) in [(0, 1), (60, 2), (120, 3)] {
|
||||
let literal = format!(
|
||||
"TQL EVAL ({timestamp_seconds}, {timestamp_seconds}, '60s') topk(1, topk({k}, {query_body}))"
|
||||
);
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
outer_expected.append(&mut rows);
|
||||
}
|
||||
let mut outer_actual = outer_native.1;
|
||||
outer_actual.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
outer_expected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(outer_actual, outer_expected, "{outer_sql}");
|
||||
|
||||
let literal_sql = format!("TQL EVAL (0, 120, '60s') topk(3, {query_body})");
|
||||
observer.clear();
|
||||
let literal_native =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal_sql, true).await).await;
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"literal topk did not run natively"
|
||||
);
|
||||
let literal_legacy =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal_sql, false).await).await;
|
||||
// The value column keeps the compiled tree's field name on this route, which the existing
|
||||
// planner names after its own rewrite, so only the rows are comparable.
|
||||
assert_eq!(literal_native.1, literal_legacy.1);
|
||||
|
||||
// A bare per-point ranking over one plain scan: nothing splits, and the whole vector is ranked
|
||||
// where every series of it is gathered rather than per partition.
|
||||
let bare_dynamic_sql = "TQL EVAL (0, 120, '60s') topk(time() / 60 + 1, native_dyn_topk)";
|
||||
observer.clear();
|
||||
let bare_dynamic_native =
|
||||
native_labelled_rows(native_tql_output(&instance, bare_dynamic_sql, true).await).await;
|
||||
let bare_dynamic_answering = observer.answering_node();
|
||||
assert_eq!(
|
||||
bare_dynamic_answering.as_deref(),
|
||||
Some("FusedSeriesExec"),
|
||||
"a bare per-point ranking must answer natively: {bare_dynamic_sql}"
|
||||
);
|
||||
assert!(
|
||||
!bare_dynamic_native.1.is_empty(),
|
||||
"a bare per-point ranking must produce rows: {bare_dynamic_sql}"
|
||||
);
|
||||
let mut bare_dynamic_expected = Vec::new();
|
||||
for (timestamp_seconds, k) in [(0, 1), (60, 2), (120, 3)] {
|
||||
let literal = format!(
|
||||
"TQL EVAL ({timestamp_seconds}, {timestamp_seconds}, '60s') topk({k}, native_dyn_topk)"
|
||||
);
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
bare_dynamic_expected.append(&mut rows);
|
||||
}
|
||||
let mut bare_dynamic_actual = bare_dynamic_native.1;
|
||||
bare_dynamic_actual.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
bare_dynamic_expected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(
|
||||
bare_dynamic_actual, bare_dynamic_expected,
|
||||
"{bare_dynamic_sql}"
|
||||
);
|
||||
|
||||
// The same ranking with a parameter that is *only* `time()`: this is the shape a per-point
|
||||
// `same_source` guard misclassifies, because the value it carries is a bare per-point constant
|
||||
// rather than an arithmetic expression over one. `k` is 0, 60 and 120 at 0s, 60s and 120s, so
|
||||
// the first point selects nothing and the other two select every series.
|
||||
let bare_time_sql = "TQL EVAL (0, 120, '60s') topk(time(), native_dyn_topk)";
|
||||
observer.clear();
|
||||
let bare_time_native =
|
||||
native_labelled_rows(native_tql_output(&instance, bare_time_sql, true).await).await;
|
||||
let bare_time_answering = observer.answering_node();
|
||||
assert_eq!(
|
||||
bare_time_answering.as_deref(),
|
||||
Some("FusedSeriesExec"),
|
||||
"a bare topk whose k is `time()` must answer natively: {bare_time_sql}"
|
||||
);
|
||||
let mut bare_time_expected = Vec::new();
|
||||
for (timestamp_seconds, k) in [(0, 0), (60, 60), (120, 120)] {
|
||||
let literal = format!(
|
||||
"TQL EVAL ({timestamp_seconds}, {timestamp_seconds}, '60s') topk({k}, native_dyn_topk)"
|
||||
);
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
bare_time_expected.append(&mut rows);
|
||||
}
|
||||
assert_eq!(
|
||||
bare_time_native.1.len(),
|
||||
bare_time_expected.len(),
|
||||
"{bare_time_sql}"
|
||||
);
|
||||
assert!(
|
||||
!bare_time_native.1.is_empty(),
|
||||
"a bare topk at k = time() must produce rows: {bare_time_sql}"
|
||||
);
|
||||
let mut bare_time_actual = bare_time_native.1;
|
||||
bare_time_actual.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
bare_time_expected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(bare_time_actual, bare_time_expected, "{bare_time_sql}");
|
||||
|
||||
// A bare `topk` has no aggregate beneath it, so there is nothing for the split to cut: the
|
||||
// whole statement is one whole-vector node over the scan, and it must answer exactly what the
|
||||
// existing planner does.
|
||||
let bare_sql = "TQL EVAL (0, 120, '60s') topk(3, native_dyn_topk)";
|
||||
observer.clear();
|
||||
let bare_native =
|
||||
native_labelled_rows(native_tql_output(&instance, bare_sql, true).await).await;
|
||||
let bare_answering = observer.answering_node();
|
||||
let bare_legacy =
|
||||
native_labelled_rows(native_tql_output(&instance, bare_sql, false).await).await;
|
||||
assert_eq!(
|
||||
bare_answering.as_deref(),
|
||||
Some("FusedSeriesExec"),
|
||||
"a bare topk above no aggregate runs natively: {bare_sql}"
|
||||
);
|
||||
assert_eq!(bare_native, bare_legacy);
|
||||
}
|
||||
|
||||
/// A `scalar()` parameter that reads its own selector runs natively: the parameter reads a metric
|
||||
/// logical table of the same physical table, so the split declines and the multi-input path answers
|
||||
/// the statement itself.
|
||||
///
|
||||
/// The parameter may read a different selector than the input (`topk(scalar(a), sum by(host)(b))`),
|
||||
/// which makes the planner prepare two sources and the compiler bind both; a metric logical table's
|
||||
/// id is part of the identity its scan reads, so the two are one table's two metrics and no split
|
||||
/// can be cut between them.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn native_promql_scalar_parameter_reads_its_own_selector() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let (standalone, observer) = native_test_instance(GreptimeDbStandaloneBuilder::new(
|
||||
"native_promql_scalar_param_selector",
|
||||
))
|
||||
.await;
|
||||
let instance = standalone.fe_instance().clone();
|
||||
execute_all(
|
||||
&instance,
|
||||
"CREATE TABLE native_scalar_param_phy (ts TIMESTAMP TIME INDEX, v DOUBLE) ENGINE=metric WITH ('physical_metric_table'='true'); \
|
||||
CREATE TABLE native_scalar_param_a (host STRING, ts TIMESTAMP TIME INDEX, v DOUBLE, PRIMARY KEY(host)) ENGINE=metric WITH ('on_physical_table'='native_scalar_param_phy'); \
|
||||
CREATE TABLE native_scalar_param_b (host STRING, ts TIMESTAMP TIME INDEX, v DOUBLE, PRIMARY KEY(host)) ENGINE=metric WITH ('on_physical_table'='native_scalar_param_phy'); \
|
||||
INSERT INTO native_scalar_param_a (host, ts, v) VALUES ('h1', 0, 2), ('h1', 60000, 4); \
|
||||
INSERT INTO native_scalar_param_b (host, ts, v) VALUES ('h1', 0, 10), ('h1', 60000, 20)",
|
||||
QueryContext::arc(),
|
||||
)
|
||||
.await;
|
||||
// `scalar(native_scalar_param_a)` is 2 at t=0 and 4 at t=60s, so the parameter differs per
|
||||
// point.
|
||||
//
|
||||
// `topk` over one metric whose `k` reads another: a per-point `k` from a second prepared
|
||||
// source, in the order the compiler binds them (`b` first, then the parameter's `a`).
|
||||
let sql = "TQL EVAL (0, 60, '60s') topk(scalar(native_scalar_param_a), sum by(host)(native_scalar_param_b))";
|
||||
observer.clear();
|
||||
let topk_native = native_labelled_rows(native_tql_output(&instance, sql, true).await).await;
|
||||
let fused = observer.fused();
|
||||
let answering = observer.answering_node();
|
||||
assert!(
|
||||
fused.is_some(),
|
||||
"a scalar() parameter that reads a selector must run natively: {sql}"
|
||||
);
|
||||
assert_eq!(
|
||||
answering.as_deref(),
|
||||
Some("FusedSeriesExec"),
|
||||
"the fused node must answer a scalar() parameter that reads a selector: {sql}"
|
||||
);
|
||||
assert_eq!(
|
||||
topk_native.1.len(),
|
||||
2,
|
||||
"one row per evaluation point: {sql}"
|
||||
);
|
||||
assert_eq!(
|
||||
topk_native.1.iter().map(|row| row.0).collect::<Vec<_>>(),
|
||||
vec![0, 60_000],
|
||||
"the expected evaluation timestamps: {sql}"
|
||||
);
|
||||
// The existing planner rejects `scalar()` over a selector, so each evaluation point is compared
|
||||
// against the same statement with the scalar's value at that point written as a literal `k`.
|
||||
let mut topk_expected = Vec::new();
|
||||
for (t, k) in [(0, 2), (60, 4)] {
|
||||
let literal =
|
||||
format!("TQL EVAL ({t}, {t}, '60s') topk({k}, sum by(host)(native_scalar_param_b))");
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal, false).await).await;
|
||||
topk_expected.append(&mut rows);
|
||||
}
|
||||
topk_expected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
let mut topk_actual = topk_native.1;
|
||||
topk_actual.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(topk_actual, topk_expected, "{sql}");
|
||||
|
||||
// The same per-point value as a `clamp` bound and as a broadcast operand.
|
||||
//
|
||||
// `clamp_max` is the bound the scalar actually decides here: `b` is 10 and 20 while `a` is 2
|
||||
// and 4, so the answer is `a` at both points and a node that ignored the parameter would answer
|
||||
// `b` instead. `clamp_min` cannot tell those apart - `b` is above `a` everywhere, so it answers
|
||||
// `b` either way - but it is the statement that used to *fail* before the wait was added, and
|
||||
// that failure is what it still pins.
|
||||
for expression in [
|
||||
"clamp_min(native_scalar_param_b, scalar(native_scalar_param_a))",
|
||||
"clamp_max(native_scalar_param_b, scalar(native_scalar_param_a))",
|
||||
"native_scalar_param_b + scalar(native_scalar_param_a)",
|
||||
] {
|
||||
let sql = format!("TQL EVAL (0, 60, '60s') {expression}");
|
||||
observer.clear();
|
||||
let native = native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
|
||||
assert_eq!(
|
||||
observer.answering_node().as_deref(),
|
||||
Some("FusedSeriesExec"),
|
||||
"cross-table dynamic scalar must run natively: {sql}"
|
||||
);
|
||||
let mut expected = Vec::new();
|
||||
for (t, value) in [(0, 2), (60, 4)] {
|
||||
let literal = expression.replace("scalar(native_scalar_param_a)", &value.to_string());
|
||||
let literal_sql = format!("TQL EVAL ({t}, {t}, '60s') {literal}");
|
||||
let (_, mut rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &literal_sql, false).await).await;
|
||||
expected.append(&mut rows);
|
||||
}
|
||||
let mut actual = native.1;
|
||||
actual.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
expected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1)));
|
||||
assert_eq!(actual, expected, "{sql}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `round` and the `clamp` functions read a varying parameter at each step, as Prometheus does. A
|
||||
/// parameter that reads no data runs per partition over the scan. The answers are Prometheus'
|
||||
/// literal ones: the existing planner does not plan a parameter that is not a number literal, so it
|
||||
|
||||
Reference in New Issue
Block a user