mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 00:34:11 +00:00
refactor(promql): decide native admission with an enumerable predicate (#10424)
Admission to the native PromQL route was decided by trial-building the whole runtime tree and discarding it, at five call sites of NativeExecutionState::try_new(...).is_err(). That is where the refusal lived, not anywhere enumerable. Replace those calls with native_tree_supported(&tree), a predicate on the compiled tree's shapes that builds no runtime: it declines the nine shape classes RuntimeNode::build rejects (raw-sample roots, interleaved inputs, scalar() over a multi-field vector, non-performable broadcasts, quantile aggregates, value-field groupings, vector ops over multi-field vectors, top-k variants, and unsupported window modifiers). try_new keeps its construction-time validation for the refusals that are not shapes. Which queries reach the native route is unchanged; the new tests assert both the predicate's verdict and that the runtime agrees. Verified: promql 1211/1211, query 1130/1130, native_promql 93/93, clippy -D warnings clean. Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
@@ -45,7 +45,7 @@ use crate::info::InfoSelector;
|
||||
use crate::map::MathOp;
|
||||
use crate::map_label::LabelRewrite;
|
||||
use crate::map_time::DateFunction;
|
||||
use crate::range_eval::{RangeFunction, RangeWindow};
|
||||
use crate::range_eval::{RangeFunction, RangeWindow, fold_offset, predict_linear_instants};
|
||||
use crate::select::SortOrder;
|
||||
use crate::vector_match::{
|
||||
BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, SetOp,
|
||||
@@ -1536,6 +1536,299 @@ impl<'de> serde::Deserialize<'de> for NativeTree {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the native runtime executes `tree`, answered from the tree itself.
|
||||
///
|
||||
/// This is the question the native route asks before it commits a statement: a tree the runtime
|
||||
/// refuses has to keep the plan it had, because a refusal raised during physical planning cannot
|
||||
/// fall back. It used to be answered by building a
|
||||
/// [`NativeExecutionState`](crate::native::execute::NativeExecutionState) over the tree and
|
||||
/// dropping it, which allocates a runtime node per node per partition only to throw it away. Every
|
||||
/// term of the answer is bound on the tree, so the answer is read from the tree instead.
|
||||
///
|
||||
/// # What is refused
|
||||
///
|
||||
/// This lists what `RuntimeNode::build` (and the root check in
|
||||
/// `NativeExecutionState::try_new`) refuses, less what [`NativeNode::validate`] has already
|
||||
/// refused: a `NativeTree` exists only through [`NativeTree::try_new`] or `Deserialize`, and both
|
||||
/// validate, so a compiled tree's per-node type contracts hold by the time this runs. What is left
|
||||
/// is the shape of the nodes, and the arithmetic each of them leaves to the runtime:
|
||||
///
|
||||
/// - a root that answers raw samples, which has no positions to answer on;
|
||||
/// - an input leaf whose series chunks may interleave, which no consumer of a leaf can order;
|
||||
/// - `scalar()` over a vector carrying more than one field;
|
||||
/// - a binary node that is neither a vector–vector operation nor a broadcast the runtime performs:
|
||||
/// two constant operands (no series to carry the result), a scalar broadcast carrying
|
||||
/// vector-matching modifiers, or `bool` on an arithmetic operator, which has no `bool` form;
|
||||
/// - the `quantile` aggregate, which has no kernel on this route at any stage;
|
||||
/// - an aggregate below `Final` whose `by` names a value field, which would fold series the
|
||||
/// existing plan keeps apart;
|
||||
/// - `topk`, `bottomk`, `limitk`, `sort` and `sort_desc` over a vector carrying more than one
|
||||
/// field, whose series have no one value to rank;
|
||||
/// - a `topk`, `bottomk`, `limitk` or `count_values` grouping that names a value field, and a
|
||||
/// `count_values` label that is empty;
|
||||
/// - a window whose `anchored` or `smoothed` modifier reads evaluated input, names a function its
|
||||
/// kernel does not correct, or carries a negative lookback;
|
||||
/// - a window whose timestamps do not fit: the fold offset an `@` anchor turns into, or the one
|
||||
/// `predict_linear` adds to a point of the grid, where the sum leaves `i64`. Not a shape, but it
|
||||
/// is decided by the grid and the window alone, so this is where it is decided (see
|
||||
/// `window_instants_fit`).
|
||||
///
|
||||
/// # What `NativeExecutionState::try_new` still decides
|
||||
///
|
||||
/// Not every refusal is decided by the tree, and the ones below stay where the runtime builds:
|
||||
///
|
||||
/// - a reservation the memory pool refuses, which no tree carries. Every site that admits a tree
|
||||
/// ran the trial build over an
|
||||
/// [`UnboundedMemoryPool`](datafusion::execution::memory_pool::UnboundedMemoryPool), which
|
||||
/// refuses nothing, and none of them builds a runtime any more, so this is not a refusal
|
||||
/// admission can meet at any site;
|
||||
/// - a tree a test assembled directly, whose nodes disagree about their types, which
|
||||
/// [`NativeNode::validate`] rejects for every compiled one.
|
||||
pub fn native_tree_supported(tree: &NativeTree) -> bool {
|
||||
// A constant root and a state root are answers the state produces itself; a root of raw
|
||||
// samples is not one, because raw samples carry their own timestamps instead of positions.
|
||||
let root_supported = match &tree.root().output {
|
||||
NativeType::Scalar(_) | NativeType::AggregateState(_) => true,
|
||||
NativeType::Samples(schema) => schema.eval_grid.is_some(),
|
||||
};
|
||||
root_supported && native_node_supported(tree.root())
|
||||
}
|
||||
|
||||
/// Whether `node` and every input below it is a shape the runtime builds.
|
||||
fn native_node_supported(node: &NativeNode) -> bool {
|
||||
stacker::maybe_grow(VALIDATE_STACK_RED_ZONE, VALIDATE_STACK_GROWTH, || {
|
||||
native_node_supported_within_stack(node)
|
||||
})
|
||||
}
|
||||
|
||||
/// [`native_node_supported`] one tree level down.
|
||||
///
|
||||
/// Every arm reads this node's own shape and recurses into exactly the inputs its variant holds,
|
||||
/// so a new [`NativeExpr`] variant does not compile until it is classified here.
|
||||
fn native_node_supported_within_stack(node: &NativeNode) -> bool {
|
||||
match &node.expr {
|
||||
NativeExpr::Input(bound) => bound.ordering != SeriesOrdering::Interleaved,
|
||||
NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => true,
|
||||
NativeExpr::VectorOf { input } | NativeExpr::Instant { input, .. } => {
|
||||
native_node_supported(input)
|
||||
}
|
||||
NativeExpr::Merge { input } | NativeExpr::Map { input, .. } => native_node_supported(input),
|
||||
NativeExpr::Timestamp { input } => native_node_supported(input),
|
||||
NativeExpr::Window { input, op } => {
|
||||
native_node_supported(input)
|
||||
&& window_modifier_supported(input, op)
|
||||
&& window_instants_fit(op)
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
} => native_node_supported(input) && parameters.iter().all(native_node_supported),
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => native_node_supported(input) && parameters.iter().all(native_node_supported),
|
||||
NativeExpr::Aggregate { input, op } => {
|
||||
native_node_supported(input) && aggregate_supported(node, input, op)
|
||||
}
|
||||
NativeExpr::Vector { input, op } => {
|
||||
native_node_supported(input)
|
||||
&& node
|
||||
.output
|
||||
.samples()
|
||||
.is_some_and(|schema| vector_op_supported(schema, op))
|
||||
}
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input,
|
||||
op,
|
||||
parameter,
|
||||
} => {
|
||||
native_node_supported(input)
|
||||
&& native_node_supported(parameter)
|
||||
// A per-point `k` ranks the same series as the constant one, one value per series.
|
||||
&& node.output.samples().is_some_and(|schema| {
|
||||
schema.fields.len() == 1 && vector_op_supported(schema, op)
|
||||
})
|
||||
}
|
||||
NativeExpr::ScalarOf { input } => {
|
||||
native_node_supported(input)
|
||||
// `scalar()` answers the sole value of a vector, so a vector of several fields has
|
||||
// more than one value to answer with.
|
||||
&& input
|
||||
.output
|
||||
.samples()
|
||||
.is_some_and(|schema| schema.fields.len() == 1)
|
||||
}
|
||||
NativeExpr::Binary {
|
||||
left,
|
||||
right,
|
||||
matching,
|
||||
} => {
|
||||
native_node_supported(left)
|
||||
&& native_node_supported(right)
|
||||
&& binary_supported(left, right, matching)
|
||||
}
|
||||
NativeExpr::Set { left, right, .. } => {
|
||||
native_node_supported(left) && native_node_supported(right)
|
||||
}
|
||||
NativeExpr::Info { base, info, .. } => {
|
||||
native_node_supported(base) && native_node_supported(info)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a binary node over `left` and `right` is a shape the runtime builds.
|
||||
///
|
||||
/// Two evaluated vectors are matched by `matching`, which the kernel takes as it is bound. Every
|
||||
/// other pair is a broadcast over one scalar side, and a broadcast has no labels to match and no
|
||||
/// series to pair: two constants leave no series to carry the result, a modifier a vector match
|
||||
/// would use has no meaning over a scalar, and arithmetic has no `bool` form.
|
||||
fn binary_supported(left: &NativeNode, right: &NativeNode, matching: &BinaryMatching) -> bool {
|
||||
let evaluated = |node: &NativeNode| {
|
||||
node.output
|
||||
.samples()
|
||||
.is_some_and(|schema| schema.eval_grid.is_some())
|
||||
};
|
||||
if evaluated(left) && evaluated(right) {
|
||||
return true;
|
||||
}
|
||||
let literal = |node: &NativeNode| matches!(&node.expr, NativeExpr::Scalar(_));
|
||||
let broadcastable = match (&left.output, &right.output) {
|
||||
(NativeType::Samples(vector), NativeType::Scalar(_))
|
||||
| (NativeType::Scalar(_), NativeType::Samples(vector)) => vector.eval_grid.is_some(),
|
||||
(NativeType::Scalar(_), NativeType::Scalar(_)) => !(literal(left) && literal(right)),
|
||||
_ => false,
|
||||
};
|
||||
broadcastable
|
||||
&& matching.labels == MatchingLabels::Ignoring(Vec::new())
|
||||
&& matching.cardinality == Cardinality::OneToOne
|
||||
&& matching.fill == FillValues::default()
|
||||
&& !(!matching.op.is_comparison() && matching.return_bool)
|
||||
}
|
||||
|
||||
/// Whether the runtime folds `op` over `input` with the layout `node` declares.
|
||||
///
|
||||
/// `quantile` has no kernel on this route, so it is refused at every stage. A `Complete` or
|
||||
/// `Partial` node folds values into groups it keys by series labels only, while `Final` merges
|
||||
/// state that was already grouped, which is why only the folding stages are checked against the
|
||||
/// schema.
|
||||
fn aggregate_supported(node: &NativeNode, input: &NativeNode, op: &BoundAggregate) -> bool {
|
||||
match op.kind {
|
||||
AggregateKind::Sum
|
||||
| AggregateKind::Count
|
||||
| AggregateKind::Avg
|
||||
| AggregateKind::Min
|
||||
| AggregateKind::Max
|
||||
| AggregateKind::Group
|
||||
| AggregateKind::Stddev
|
||||
| AggregateKind::Stdvar => {}
|
||||
AggregateKind::Quantile { .. } => return false,
|
||||
}
|
||||
if op.stage == AggregateStage::Final {
|
||||
return true;
|
||||
}
|
||||
node.output
|
||||
.samples()
|
||||
.or_else(|| input.output.samples())
|
||||
.is_some_and(|schema| grouping_supported(schema, &op.group))
|
||||
}
|
||||
|
||||
/// Whether `op` is an operation the vector kernel runs over a vector of `schema`.
|
||||
fn vector_op_supported(schema: &SeriesSchema, op: &BoundVectorOp) -> bool {
|
||||
// These five rank or order one value per series, so a vector whose samples carry several
|
||||
// fields has no one value to rank.
|
||||
let one_field = matches!(
|
||||
op,
|
||||
BoundVectorOp::TopK { .. }
|
||||
| BoundVectorOp::LimitK { .. }
|
||||
| BoundVectorOp::Sort(SortOrder::Asc | SortOrder::Desc)
|
||||
);
|
||||
if one_field && schema.fields.len() != 1 {
|
||||
return false;
|
||||
}
|
||||
match op {
|
||||
BoundVectorOp::TopK { group, .. } | BoundVectorOp::LimitK { group, .. } => {
|
||||
grouping_supported(schema, group)
|
||||
}
|
||||
BoundVectorOp::CountValues { label, group } => {
|
||||
let names = group_names(group);
|
||||
crate::aggregate::validate_count_values(
|
||||
schema,
|
||||
label.as_ref(),
|
||||
kernel_grouping(group, &names),
|
||||
)
|
||||
.is_ok()
|
||||
}
|
||||
BoundVectorOp::LimitRatio { .. }
|
||||
| BoundVectorOp::Sort(_)
|
||||
| BoundVectorOp::Absent { .. }
|
||||
| BoundVectorOp::Histogram(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the kernel folds `schema`'s series the way `group` asks.
|
||||
///
|
||||
/// The kernel validates its grouping by the schema alone: the group is a set of series labels, and
|
||||
/// a `by` naming one of the schema's fields would group by a value that varies between samples.
|
||||
/// This is the check `FloatAggregator::try_new_budgeted` and `VectorRuntime::try_new` run, spelled
|
||||
/// once so a reader of either name finds the same rule.
|
||||
fn grouping_supported(schema: &SeriesSchema, group: &Grouping) -> bool {
|
||||
let names = group_names(group);
|
||||
crate::aggregate::validate_vector_grouping(schema, kernel_grouping(group, &names)).is_ok()
|
||||
}
|
||||
|
||||
/// The labels `group` names, which is what the kernel's grouping is built over.
|
||||
fn group_names(group: &Grouping) -> Vec<&str> {
|
||||
match group {
|
||||
Grouping::By(names) | Grouping::Without(names) => names.iter().map(AsRef::as_ref).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `group` spelled with the kernel's own [`Grouping`](crate::aggregate::Grouping), over the labels
|
||||
/// [`group_names`] reports for it: `by` and `without` are not interchangeable to the kernel.
|
||||
fn kernel_grouping<'a>(group: &Grouping, names: &'a [&'a str]) -> crate::aggregate::Grouping<'a> {
|
||||
match group {
|
||||
Grouping::By(_) => crate::aggregate::Grouping::By(names),
|
||||
Grouping::Without(_) => crate::aggregate::Grouping::Without(names),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the runtime evaluates `op` over `input`, with or without a range modifier.
|
||||
///
|
||||
/// An `anchored` or `smoothed` window reads samples around the window to pin its boundary values,
|
||||
/// which only a raw sample stream can offer and only a kernel that corrects its boundaries reads.
|
||||
/// That is one rule, not two: [`check_modifier`](crate::range_eval::check_modifier) states it for
|
||||
/// [`RangeEvaluator::try_new`](crate::range_eval::RangeEvaluator::try_new), and it is asked here
|
||||
/// over the schema the tree binds, so a modifier a runtime refuses is one admission declines.
|
||||
fn window_modifier_supported(input: &NativeNode, op: &BoundWindow) -> bool {
|
||||
// Without a modifier there is nothing for the rule to decide.
|
||||
let Some(modifier) = op.window.modifier else {
|
||||
return true;
|
||||
};
|
||||
input.output.samples().is_some_and(|schema| {
|
||||
crate::range_eval::check_modifier(schema, modifier, op.function).is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the runtime can compute the timestamps `op` reads.
|
||||
///
|
||||
/// Unlike the window itself, which a point of the grid opens alone, these timestamps are sums of
|
||||
/// the window's own parts: an `@` anchor becomes an offset from the grid's first point, and
|
||||
/// `predict_linear` adds that offset to the first and last point of the grid. Every term of either
|
||||
/// sum is bound on the node, but a sum of two that fit can still leave `i64`, and the runtime
|
||||
/// refuses the window when one does. Asking [`fold_offset`] and [`predict_linear_instants`] here
|
||||
/// rather than repeating their arithmetic keeps the two verdicts the same one: a query whose only
|
||||
/// refusal is this one keeps the plan it had instead of failing while the runtime is built.
|
||||
fn window_instants_fit(op: &BoundWindow) -> bool {
|
||||
let grid = op.output_grid.spec();
|
||||
let Ok(fold_offset) = fold_offset(grid, op.window) else {
|
||||
return false;
|
||||
};
|
||||
match op.function {
|
||||
// Only `predict_linear` reads the instant it reports; every other function reads the
|
||||
// window it opens, which the grid alone decides.
|
||||
RangeFunction::PredictLinear(_) => predict_linear_instants(grid, fold_offset).is_ok(),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Stack one [`NativeNode::validate`] level needs below it before the check grows its stack.
|
||||
///
|
||||
/// The check recurses once per tree level, and a debug build's frame for one level is large:
|
||||
|
||||
@@ -30,7 +30,7 @@ use crate::extension_plan::{FusedSeries, FusedSeriesBinding};
|
||||
use crate::histogram_eval::NativeHistogramFunction;
|
||||
use crate::info::InfoSelector;
|
||||
use crate::map::MathOp;
|
||||
use crate::range_eval::{RangeFunction, RangeWindow};
|
||||
use crate::range_eval::{RangeFunction, RangeModifier, RangeWindow};
|
||||
use crate::vector_match::{
|
||||
BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, SetOp,
|
||||
};
|
||||
@@ -244,6 +244,24 @@ fn map_over(inner: NativeNode) -> NativeNode {
|
||||
}
|
||||
}
|
||||
|
||||
/// An aggregate over `inner` reporting `out`, folding it with `kind`, `group` and `stage` as
|
||||
/// written.
|
||||
fn aggregate_with(
|
||||
inner: NativeNode,
|
||||
out: Arc<EvalGrid>,
|
||||
kind: AggregateKind,
|
||||
group: Grouping,
|
||||
stage: AggregateStage,
|
||||
) -> NativeNode {
|
||||
NativeNode {
|
||||
output: evaluated(fields(inner_field_count(&inner)), Arc::clone(&out)),
|
||||
expr: NativeExpr::Aggregate {
|
||||
input: Box::new(inner),
|
||||
op: BoundAggregate { kind, group, stage },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn add() -> BinaryMatching {
|
||||
BinaryMatching {
|
||||
op: BinaryOp::Add,
|
||||
@@ -2929,3 +2947,668 @@ fn independently_allocated_binding_mismatch_is_rejected() {
|
||||
}));
|
||||
assert!(mismatch.validate().is_err());
|
||||
}
|
||||
|
||||
/// Asserts the admission verdict for `node`, and that the runtime's own construction answers the
|
||||
/// same: a predicate that stopped agreeing with it would either admit a tree the executor cannot
|
||||
/// build, or decline a shape the executor can run.
|
||||
fn assert_admission(node: NativeNode, admitted: bool, shape: &str) {
|
||||
use datafusion::execution::memory_pool::UnboundedMemoryPool;
|
||||
|
||||
use crate::native::execute::NativeExecutionState;
|
||||
|
||||
let tree = NativeTree::try_new(node)
|
||||
.unwrap_or_else(|error| panic!("{shape} is not a tree the compiler produces: {error}"));
|
||||
assert_eq!(super::native_tree_supported(&tree), admitted, "{shape}");
|
||||
assert_eq!(
|
||||
NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None)
|
||||
.is_ok(),
|
||||
admitted,
|
||||
"{shape}: the runtime does not agree with the admission predicate"
|
||||
);
|
||||
}
|
||||
|
||||
/// `outer` over `inner`, with `function` and `modifier` bound as written.
|
||||
fn window_with(
|
||||
inner: NativeNode,
|
||||
input_grid: Option<Arc<EvalGrid>>,
|
||||
out: Arc<EvalGrid>,
|
||||
function: RangeFunction,
|
||||
modifier: Option<RangeModifier>,
|
||||
) -> NativeNode {
|
||||
NativeNode {
|
||||
expr: NativeExpr::Window {
|
||||
input: Box::new(inner),
|
||||
op: BoundWindow {
|
||||
function,
|
||||
window: RangeWindow {
|
||||
range_ms: 300_000,
|
||||
offset_ms: 0,
|
||||
anchor_ms: None,
|
||||
modifier,
|
||||
},
|
||||
input_grid,
|
||||
output_grid: Arc::clone(&out),
|
||||
},
|
||||
},
|
||||
output: evaluated(Arc::new([]), out),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every shape the runtime refuses is refused here, and each one is a shape a tree can hold.
|
||||
#[test]
|
||||
fn native_tree_supported_declines_the_shapes_the_runtime_refuses() {
|
||||
use crate::select::SortOrder;
|
||||
|
||||
let out = grid(0, 60_000, 4);
|
||||
// A leaf whose chunks of different series may interleave: no consumer of a leaf orders them.
|
||||
let interleaved = NativeNode {
|
||||
expr: NativeExpr::Input(BoundInput {
|
||||
output: evaluated(Arc::new([]), Arc::clone(&out)),
|
||||
source_index: 0,
|
||||
ordering: SeriesOrdering::Interleaved,
|
||||
}),
|
||||
output: evaluated(Arc::new([]), Arc::clone(&out)),
|
||||
};
|
||||
let one_field = || window_fields(input_with_fields(1), Arc::clone(&out), 1);
|
||||
let two_fields = || window_fields(input_with_fields(2), Arc::clone(&out), 2);
|
||||
let vector = |inner: NativeNode, op| NativeNode {
|
||||
output: inner.output.clone(),
|
||||
expr: NativeExpr::Vector {
|
||||
input: Box::new(inner),
|
||||
op,
|
||||
},
|
||||
};
|
||||
let cases = vec![
|
||||
("a root of raw samples", input()),
|
||||
(
|
||||
"an input leaf whose chunks may interleave",
|
||||
map_over(interleaved),
|
||||
),
|
||||
(
|
||||
"scalar() over a vector of two fields",
|
||||
NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(two_fields()),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"two constant operands",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
right: Box::new(scalar_on(2.0, Arc::clone(&out))),
|
||||
matching: add(),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a scalar broadcast carrying a matching modifier",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
matching: BinaryMatching {
|
||||
labels: MatchingLabels::Ignoring(vec!["host".into()]),
|
||||
..add()
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a scalar broadcast of bool arithmetic",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
matching: BinaryMatching {
|
||||
return_bool: true,
|
||||
..add()
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a scalar broadcast carrying a cardinality modifier",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
matching: BinaryMatching {
|
||||
cardinality: Cardinality::ManyToOne(Vec::new()),
|
||||
..add()
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a scalar broadcast filling missing values",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
matching: BinaryMatching {
|
||||
fill: FillValues {
|
||||
lhs: Some(0.0),
|
||||
rhs: None,
|
||||
},
|
||||
..add()
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"the quantile aggregate",
|
||||
aggregate_with(
|
||||
one_field(),
|
||||
Arc::clone(&out),
|
||||
AggregateKind::Quantile { quantile: 0.9 },
|
||||
Grouping::By(Arc::new(["job".into()])),
|
||||
AggregateStage::Complete,
|
||||
),
|
||||
),
|
||||
(
|
||||
"an aggregate whose `by` names a value field",
|
||||
aggregate_with(
|
||||
one_field(),
|
||||
Arc::clone(&out),
|
||||
AggregateKind::Sum,
|
||||
Grouping::By(Arc::new(["f0".into()])),
|
||||
AggregateStage::Complete,
|
||||
),
|
||||
),
|
||||
(
|
||||
"topk over a vector of two fields",
|
||||
vector(
|
||||
two_fields(),
|
||||
BoundVectorOp::TopK {
|
||||
k: 1.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::new([])),
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
"topk grouped by a value field",
|
||||
vector(
|
||||
one_field(),
|
||||
BoundVectorOp::TopK {
|
||||
k: 1.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::new(["f0".into()])),
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
"sort over a vector of two fields",
|
||||
vector(two_fields(), BoundVectorOp::Sort(SortOrder::Asc)),
|
||||
),
|
||||
(
|
||||
"count_values under an empty label",
|
||||
vector(
|
||||
one_field(),
|
||||
BoundVectorOp::CountValues {
|
||||
label: Arc::from(""),
|
||||
group: Grouping::By(Arc::new([])),
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
"a parameterized topk over a vector of two fields",
|
||||
NativeNode {
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(two_fields()),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::new([])),
|
||||
},
|
||||
parameter: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
},
|
||||
output: evaluated(fields(2), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"an anchored window over evaluated input",
|
||||
window_with(
|
||||
input_on(Arc::clone(&out)),
|
||||
Some(Arc::clone(&out)),
|
||||
Arc::clone(&out),
|
||||
RangeFunction::Rate,
|
||||
Some(RangeModifier::Anchored {
|
||||
lookback_ms: 60_000,
|
||||
}),
|
||||
),
|
||||
),
|
||||
(
|
||||
"a smoothed window over a function its kernel does not correct",
|
||||
window_with(
|
||||
input(),
|
||||
None,
|
||||
Arc::clone(&out),
|
||||
RangeFunction::SumOverTime,
|
||||
Some(RangeModifier::Smoothed {
|
||||
lookback_ms: 60_000,
|
||||
}),
|
||||
),
|
||||
),
|
||||
(
|
||||
"an anchored window over a function its kernel does not correct",
|
||||
window_with(
|
||||
input(),
|
||||
None,
|
||||
Arc::clone(&out),
|
||||
RangeFunction::SumOverTime,
|
||||
Some(RangeModifier::Anchored {
|
||||
lookback_ms: 60_000,
|
||||
}),
|
||||
),
|
||||
),
|
||||
(
|
||||
"a window with a negative modifier lookback",
|
||||
window_with(
|
||||
input(),
|
||||
None,
|
||||
Arc::clone(&out),
|
||||
RangeFunction::Rate,
|
||||
Some(RangeModifier::Anchored { lookback_ms: -1 }),
|
||||
),
|
||||
),
|
||||
];
|
||||
for (shape, node) in cases {
|
||||
assert_admission(node, false, shape);
|
||||
}
|
||||
}
|
||||
|
||||
/// A refusal below the root is still a refusal: admission reads the whole tree, not its root.
|
||||
///
|
||||
/// Each case below holds a `quantile` — which has no kernel on this route, at any stage or depth —
|
||||
/// under a node whose own shape the runtime does build. A predicate that stopped recursing into
|
||||
/// one of these variants' inputs, or into a parameterised node's parameters, would admit a tree
|
||||
/// the runtime then refuses, and a refusal raised while the runtime is built cannot fall back: the
|
||||
/// statement loses the plan it had instead of keeping it.
|
||||
#[test]
|
||||
fn native_tree_supported_declines_a_refused_shape_nested_under_another() {
|
||||
let out = grid(0, 60_000, 4);
|
||||
let one_field = || window_fields(input_with_fields(1), Arc::clone(&out), 1);
|
||||
// The declined shape every case nests, one level down from a node the runtime builds.
|
||||
let quantile = || {
|
||||
aggregate_with(
|
||||
one_field(),
|
||||
Arc::clone(&out),
|
||||
AggregateKind::Quantile { quantile: 0.9 },
|
||||
Grouping::By(Arc::new(["job".into()])),
|
||||
AggregateStage::Complete,
|
||||
)
|
||||
};
|
||||
// The info side `info` reads, which `validate` requires to be the times of an instant
|
||||
// selection; only the base side of the node carries the quantile.
|
||||
let info_times = || {
|
||||
merge_over(NativeNode {
|
||||
expr: NativeExpr::Instant {
|
||||
input: Box::new(input_with_fields(1)),
|
||||
op: BoundInstant {
|
||||
lookback_ms: 300_000,
|
||||
offset_ms: 0,
|
||||
at_ms: None,
|
||||
value: InstantValue::InfoTimestamp,
|
||||
output_grid: Arc::clone(&out),
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
})
|
||||
};
|
||||
let cases = vec![
|
||||
(
|
||||
"a quantile under a binary operation",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(quantile()),
|
||||
right: Box::new(one_field()),
|
||||
matching: add(),
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a quantile under a set operation",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Set {
|
||||
left: Box::new(quantile()),
|
||||
right: Box::new(one_field()),
|
||||
op: super::BoundSetOp {
|
||||
op: SetOp::Or,
|
||||
labels: MatchingLabels::Ignoring(Vec::new()),
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a quantile under info",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Info {
|
||||
base: Box::new(quantile()),
|
||||
info: Box::new(info_times()),
|
||||
selector: InfoSelector {
|
||||
name_matchers: Vec::new(),
|
||||
data_labels: Vec::new(),
|
||||
keep_unmatched: true,
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a quantile under an aggregate",
|
||||
aggregate_with(
|
||||
quantile(),
|
||||
Arc::clone(&out),
|
||||
AggregateKind::Sum,
|
||||
Grouping::By(Arc::new(["job".into()])),
|
||||
AggregateStage::Complete,
|
||||
),
|
||||
),
|
||||
(
|
||||
"a quantile under a parameterized topk",
|
||||
NativeNode {
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(quantile()),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::new([])),
|
||||
},
|
||||
parameter: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
];
|
||||
for (shape, node) in cases {
|
||||
assert_admission(node, false, shape);
|
||||
}
|
||||
}
|
||||
|
||||
/// A window whose timestamps do not fit is refused by the runtime, so admission refuses it too.
|
||||
///
|
||||
/// These two are not shapes and every term of their arithmetic fits: an `@` anchor is folded into
|
||||
/// an offset from the grid's first point, which is a difference of two timestamps, and
|
||||
/// `predict_linear` adds that offset to the first and last point of the grid. A tree carrying
|
||||
/// either is still one the runtime cannot build, and admitting it is what made a query the
|
||||
/// existing plan answered fail while its runtime was built instead of keeping that plan, because a
|
||||
/// refusal raised during physical planning cannot fall back.
|
||||
#[test]
|
||||
fn native_tree_supported_declines_the_instants_that_overflow() {
|
||||
// The first point of a statement reaching far into the future, a step apart: this is the grid
|
||||
// `predict_linear(some_metric[1m] @ 0, 60)` is planned over, whose anchor makes the fold offset
|
||||
// the grid's own start and whose horizon then adds the two.
|
||||
let far = grid(5_000_000_000_000_000_000, 5_000, 3);
|
||||
let window = |function: RangeFunction, anchor_ms: Option<i64>, out: Arc<EvalGrid>| NativeNode {
|
||||
expr: NativeExpr::Window {
|
||||
input: Box::new(input()),
|
||||
op: BoundWindow {
|
||||
function,
|
||||
window: RangeWindow {
|
||||
range_ms: 60_000,
|
||||
offset_ms: 0,
|
||||
anchor_ms,
|
||||
modifier: None,
|
||||
},
|
||||
input_grid: None,
|
||||
output_grid: Arc::clone(&out),
|
||||
},
|
||||
},
|
||||
output: evaluated(Arc::new([]), out),
|
||||
};
|
||||
// The fold offset is 5e18 ms and `predict_linear` adds it to a point of the grid: 1e19 ms is
|
||||
// what an `i64` cannot hold, so the kernel would read an instant that does not exist.
|
||||
let horizon = window(
|
||||
RangeFunction::PredictLinear(60.0),
|
||||
Some(0),
|
||||
Arc::clone(&far),
|
||||
);
|
||||
assert_admission(
|
||||
horizon,
|
||||
false,
|
||||
"a predict_linear horizon over the grid that overflows",
|
||||
);
|
||||
// The same overflow without `predict_linear`: `@ -5e18` over that grid is a fold offset of
|
||||
// 1e19 ms on its own, so even `rate` reads a window the runtime cannot place.
|
||||
let anchored = window(
|
||||
RangeFunction::Rate,
|
||||
Some(-5_000_000_000_000_000_000),
|
||||
Arc::clone(&far),
|
||||
);
|
||||
assert_admission(anchored, false, "an `@` anchor whose fold offset overflows");
|
||||
// The same shapes over a grid near the epoch fit, so the check does not narrow the route.
|
||||
let fitting = window(
|
||||
RangeFunction::PredictLinear(60.0),
|
||||
Some(0),
|
||||
grid(0, 5_000, 3),
|
||||
);
|
||||
assert_admission(fitting, true, "a predict_linear horizon near the epoch");
|
||||
}
|
||||
|
||||
/// Every shape the runtime builds is admitted, so the predicate does not narrow the route.
|
||||
#[test]
|
||||
fn native_tree_supported_admits_the_shapes_the_runtime_builds() {
|
||||
use crate::select::SortOrder;
|
||||
|
||||
let out = grid(0, 60_000, 4);
|
||||
let one_field = || window_fields(input_with_fields(1), Arc::clone(&out), 1);
|
||||
let two_fields = || window_fields(input_with_fields(2), Arc::clone(&out), 2);
|
||||
let vector = |inner: NativeNode, op| NativeNode {
|
||||
output: inner.output.clone(),
|
||||
expr: NativeExpr::Vector {
|
||||
input: Box::new(inner),
|
||||
op,
|
||||
},
|
||||
};
|
||||
let cases = vec![
|
||||
("an evaluated input leaf", input_on(Arc::clone(&out))),
|
||||
("a constant root", scalar_on(3.0, Arc::clone(&out))),
|
||||
(
|
||||
"an evaluation-time root",
|
||||
NativeNode {
|
||||
expr: NativeExpr::EvalTime { date: None },
|
||||
output: NativeType::Scalar(Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
("a window over raw samples", window_over(input(), None)),
|
||||
(
|
||||
"an instant selection over raw samples",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Instant {
|
||||
input: Box::new(input()),
|
||||
op: BoundInstant {
|
||||
lookback_ms: 300_000,
|
||||
offset_ms: 0,
|
||||
at_ms: None,
|
||||
value: InstantValue::Sample,
|
||||
output_grid: Arc::clone(&out),
|
||||
},
|
||||
},
|
||||
output: evaluated(Arc::new([]), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a merge of an evaluated vector",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Merge {
|
||||
input: Box::new(one_field()),
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a map over an evaluated vector",
|
||||
map_over(input_on(Arc::clone(&out))),
|
||||
),
|
||||
(
|
||||
"scalar() over a vector of one field",
|
||||
NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(one_field()),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"vector(s)",
|
||||
NativeNode {
|
||||
expr: NativeExpr::VectorOf {
|
||||
input: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"an aggregate whose `by` names a label",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Aggregate {
|
||||
input: Box::new(one_field()),
|
||||
op: BoundAggregate {
|
||||
kind: AggregateKind::Sum,
|
||||
group: Grouping::By(Arc::new(["job".into()])),
|
||||
stage: AggregateStage::Complete,
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a partial aggregate root",
|
||||
partial_aggregate(window_over(input(), None)),
|
||||
),
|
||||
(
|
||||
"a final aggregate over partial state",
|
||||
final_aggregate(partial_aggregate(window_over(input(), None))),
|
||||
),
|
||||
(
|
||||
"two evaluated vectors",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(one_field()),
|
||||
matching: add(),
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"two evaluated vectors under a cardinality modifier",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(one_field()),
|
||||
matching: BinaryMatching {
|
||||
cardinality: Cardinality::ManyToOne(Vec::new()),
|
||||
labels: MatchingLabels::On(vec!["job".into()]),
|
||||
..add()
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a scalar broadcast",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(scalar_on(1.0, Arc::clone(&out))),
|
||||
matching: add(),
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"a set operation over two evaluated vectors",
|
||||
NativeNode {
|
||||
expr: NativeExpr::Set {
|
||||
left: Box::new(one_field()),
|
||||
right: Box::new(one_field()),
|
||||
op: super::BoundSetOp {
|
||||
op: SetOp::Or,
|
||||
labels: MatchingLabels::Ignoring(Vec::new()),
|
||||
},
|
||||
},
|
||||
output: evaluated(fields(1), Arc::clone(&out)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"topk over a vector of one field",
|
||||
vector(
|
||||
one_field(),
|
||||
BoundVectorOp::TopK {
|
||||
k: 1.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::new([])),
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
"count_values naming its own value field",
|
||||
vector(
|
||||
one_field(),
|
||||
BoundVectorOp::CountValues {
|
||||
label: Arc::from("f0"),
|
||||
group: Grouping::By(Arc::new(["f0".into()])),
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
"sort_by_label over a vector of two fields",
|
||||
vector(
|
||||
two_fields(),
|
||||
BoundVectorOp::Sort(SortOrder::ByLabel(vec!["f0".into()])),
|
||||
),
|
||||
),
|
||||
(
|
||||
"limit_ratio over a vector of two fields",
|
||||
vector(two_fields(), BoundVectorOp::LimitRatio { ratio: 0.5 }),
|
||||
),
|
||||
(
|
||||
"absent over a vector of two fields",
|
||||
vector(
|
||||
two_fields(),
|
||||
BoundVectorOp::Absent {
|
||||
labels: datatypes::series::SeriesLabels::try_new(None, Vec::new()).unwrap(),
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
"a window without a modifier",
|
||||
window_with(input(), None, Arc::clone(&out), RangeFunction::Rate, None),
|
||||
),
|
||||
(
|
||||
"a smoothed window over a function its kernel corrects",
|
||||
window_with(
|
||||
input(),
|
||||
None,
|
||||
Arc::clone(&out),
|
||||
RangeFunction::Rate,
|
||||
Some(RangeModifier::Smoothed {
|
||||
lookback_ms: 60_000,
|
||||
}),
|
||||
),
|
||||
),
|
||||
];
|
||||
for (shape, node) in cases {
|
||||
assert_admission(node, true, shape);
|
||||
}
|
||||
}
|
||||
|
||||
/// The field count `inner` carries, which a complete aggregate reports unchanged.
|
||||
fn inner_field_count(inner: &NativeNode) -> usize {
|
||||
inner
|
||||
.output
|
||||
.samples()
|
||||
.map_or(0, |schema| schema.fields.len())
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ use common_query::native_histogram::native_histogram_value_type;
|
||||
use common_query::series_frame::raw_series_frame_schema_with_ordering;
|
||||
use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit as ArrowTimeUnit};
|
||||
use datafusion::common::{DFSchema, DFSchemaRef};
|
||||
use datafusion::execution::memory_pool::UnboundedMemoryPool;
|
||||
use datafusion::logical_expr::{Extension, LogicalPlan};
|
||||
use datatypes::data_type::DataType as _;
|
||||
use datatypes::series::SeriesOrdering;
|
||||
@@ -42,7 +41,7 @@ use crate::extension_plan::{
|
||||
FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, InputLocality, RawSeriesSource,
|
||||
input_locality, tree_source_fields,
|
||||
};
|
||||
use crate::native::execute::NativeExecutionState;
|
||||
use crate::native::plan::native_tree_supported;
|
||||
use crate::native::plan::{
|
||||
BoundAggregate, BoundInput, BoundVectorOp, Grouping, METRIC_NAME, NativeExpr, NativeNode,
|
||||
NativeTree, NativeType, reads_across_series,
|
||||
@@ -850,8 +849,7 @@ fn build_evaluated_segment(
|
||||
return Some((plan, tags));
|
||||
}
|
||||
let output_tags = output_tags_of(tree.root(), &local_tags)?;
|
||||
if NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None).is_err()
|
||||
{
|
||||
if !native_tree_supported(&tree) {
|
||||
return None;
|
||||
}
|
||||
let frame = FusedSeries::series_frame_schema(&tree).ok()?;
|
||||
@@ -942,8 +940,7 @@ fn build_inner_segment(
|
||||
let time_index = per_source.get(*sources.first()?)?.1.clone();
|
||||
(children, None, time_index, output_tags.clone())
|
||||
};
|
||||
if NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None).is_err()
|
||||
{
|
||||
if !native_tree_supported(&tree) {
|
||||
return None;
|
||||
}
|
||||
let frame = FusedSeries::series_frame_schema(&tree).ok()?;
|
||||
@@ -1192,9 +1189,7 @@ pub fn segment_tree(
|
||||
// Nothing below can fall back once physical planning starts, so every tree this route commits
|
||||
// to has to be one the runtime executes. A shape the runtime refuses - a grouping by a value
|
||||
// field, an operation with no kernel - therefore keeps the plan it had.
|
||||
if NativeExecutionState::try_new(&parent, Arc::new(UnboundedMemoryPool::default()), None)
|
||||
.is_err()
|
||||
{
|
||||
if !native_tree_supported(&parent) {
|
||||
return None;
|
||||
}
|
||||
let output_schema = match format {
|
||||
|
||||
@@ -612,16 +612,7 @@ impl RangeEvaluator {
|
||||
let fold_offset = fold_offset(grid.spec(), window)?;
|
||||
let step_invariant = !matches!(function, RangeFunction::PredictLinear(_));
|
||||
if !step_invariant {
|
||||
let spec = grid.spec();
|
||||
let last =
|
||||
i128::from(spec.start_ms) + i128::from(spec.step_ms) * (i128::from(spec.len) - 1);
|
||||
for eval_ts in [i128::from(spec.start_ms), last] {
|
||||
if i64::try_from(eval_ts + i128::from(fold_offset)).is_err() {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"predict_linear evaluation time {eval_ts} overflows with offset {fold_offset} ms"
|
||||
)));
|
||||
}
|
||||
}
|
||||
predict_linear_instants(grid.spec(), fold_offset)?;
|
||||
}
|
||||
// A modified window corrects counters over the samples it holds rather than through the
|
||||
// reset index, and has no batch form.
|
||||
@@ -1505,7 +1496,11 @@ impl RowEvaluator<'_> {
|
||||
/// Rejects a modifier on a function Prometheus does not allow it with, as Prometheus rejects the
|
||||
/// query (`AnchoredSafeFunctions`, `SmoothedSafeFunctions`), and on evaluated input, which reads
|
||||
/// no raw samples around its windows.
|
||||
fn check_modifier(
|
||||
///
|
||||
/// `pub(crate)` because it is a rule about the input's schema and the window alone, so the native
|
||||
/// route's admission predicate (`native_tree_supported`) asks it rather than a second copy of it,
|
||||
/// the way it asks [`fold_offset`].
|
||||
pub(crate) fn check_modifier(
|
||||
input: &SeriesSchema,
|
||||
modifier: RangeModifier,
|
||||
function: RangeFunction,
|
||||
@@ -1551,7 +1546,10 @@ fn check_modifier(
|
||||
|
||||
/// The offset samples are shifted by: the selector offset, or for `@` the offset that places
|
||||
/// the anchored window at the first grid point.
|
||||
fn fold_offset(grid: GridSpec, window: RangeWindow) -> Result<i64> {
|
||||
///
|
||||
/// `pub(crate)` because whether it fits is decided from the grid and the window alone, so the
|
||||
/// native route's admission predicate asks the same arithmetic rather than a second copy of it.
|
||||
pub(crate) fn fold_offset(grid: GridSpec, window: RangeWindow) -> Result<i64> {
|
||||
let Some(anchor) = window.anchor_ms else {
|
||||
return Ok(window.offset_ms);
|
||||
};
|
||||
@@ -1566,6 +1564,30 @@ fn fold_offset(grid: GridSpec, window: RangeWindow) -> Result<i64> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Checks that `predict_linear` can read the instant of every point of `grid` once shifted by
|
||||
/// `fold_offset`.
|
||||
///
|
||||
/// Unlike every other range function, which reads the window a grid point opens alone,
|
||||
/// `predict_linear` reads the instant it reports as well, and reports it shifted by the fold
|
||||
/// offset. The first and the last point are the ones checked: the shift is the same at every point
|
||||
/// and the grid's points are ordered, so what fits at both ends fits between them.
|
||||
///
|
||||
/// The refusal is arithmetic rather than shape, but it is decided by the grid and the window alone,
|
||||
/// which is why the native route's admission predicate reaches it too
|
||||
/// (`native_tree_supported`): a query refused only here has to keep the plan it had instead of
|
||||
/// failing while its runtime is built.
|
||||
pub(crate) fn predict_linear_instants(grid: GridSpec, fold_offset: i64) -> Result<()> {
|
||||
let last = i128::from(grid.start_ms) + i128::from(grid.step_ms) * (i128::from(grid.len) - 1);
|
||||
for eval_ts in [i128::from(grid.start_ms), last] {
|
||||
if i64::try_from(eval_ts + i128::from(fold_offset)).is_err() {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"predict_linear evaluation time {eval_ts} overflows with offset {fold_offset} ms"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The inner grid of a subquery read by `window` from the outer grid `outer`: the multiples of
|
||||
/// `step_ms` after `start - F - range` up to the last window end, where `F` is the fold offset
|
||||
/// of `window`; with `@`, the only window end is `start - F`. `None` when no multiple falls in
|
||||
|
||||
@@ -41,7 +41,6 @@ use common_query::series_frame::raw_series_frame_schema_with_ordering;
|
||||
use datafusion::arrow::datatypes::DataType as ArrowDataType;
|
||||
use datafusion::common::{DFSchema, DFSchemaRef};
|
||||
use datafusion::error::DataFusionError;
|
||||
use datafusion::execution::memory_pool::UnboundedMemoryPool;
|
||||
use datafusion::logical_expr::UserDefinedLogicalNode;
|
||||
use datafusion::logical_expr::{EmptyRelation, Extension, LogicalPlan, TableScanBuilder, Union};
|
||||
use datafusion::physical_plan::ExecutionPlan;
|
||||
@@ -55,10 +54,9 @@ use promql::extension_plan::{
|
||||
FieldKind, FusedOutputFormat, FusedSeries, FusedSeriesBinding, FusedSeriesExec, InputLocality,
|
||||
RawSeriesSource, input_locality, matching_operands_drop_key,
|
||||
};
|
||||
use promql::native::execute::NativeExecutionState;
|
||||
use promql::native::plan::{
|
||||
AggregateKind, BoundInput, BoundVectorOp, NativeExpr, NativeNode, NativeTree, NativeType,
|
||||
binary_output_field_kind, fused_output_tags, uniform_field_kind,
|
||||
binary_output_field_kind, fused_output_tags, native_tree_supported, uniform_field_kind,
|
||||
};
|
||||
use promql::native::segment::{
|
||||
self, NATIVE_ORDERING, canonical_boundary_stages, canonical_local_prefix,
|
||||
@@ -523,9 +521,7 @@ impl PromPlanner {
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None)
|
||||
.is_err()
|
||||
{
|
||||
if !native_tree_supported(&tree) {
|
||||
return Ok(None);
|
||||
}
|
||||
// The expression names the value columns, and it is the label the node reports.
|
||||
@@ -1468,13 +1464,12 @@ impl PromPlanner {
|
||||
if multi_input && groups_by_metric_name(root) {
|
||||
return Ok(None);
|
||||
}
|
||||
// Catch construction-time refusals before choosing native execution: errors raised during
|
||||
// physical planning cannot fall back. This does not cover data-dependent errors raised while
|
||||
// executing the tree.
|
||||
if (multi_input || per_source_inputs)
|
||||
&& NativeExecutionState::try_new(&tree, Arc::new(UnboundedMemoryPool::default()), None)
|
||||
.is_err()
|
||||
{
|
||||
// Catch the shape refusals before choosing native execution: errors raised during physical
|
||||
// planning cannot fall back. This includes unsupported aggregates and `topk`, `limitk` or
|
||||
// CountValues groupings; it does not cover data-dependent errors raised while executing the
|
||||
// tree. A plan split over regions commits to the fused node without the split or a segment
|
||||
// above it, so it checks itself the same way before that choice.
|
||||
if (multi_input || per_source_inputs) && !native_tree_supported(&tree) {
|
||||
return Ok(None);
|
||||
}
|
||||
// A tree that ends in a canonical merge answers one series per label set, and the pieces of one
|
||||
|
||||
@@ -3407,6 +3407,83 @@ async fn native_fused_partitioned_table_label_rewrite_falls_back() {
|
||||
);
|
||||
}
|
||||
|
||||
/// A `predict_linear` horizon whose instants do not fit keeps the existing plan.
|
||||
///
|
||||
/// A window folds its `@` anchor into an offset from the grid's first point, and `predict_linear`
|
||||
/// is the one range function that adds that offset to a point of the grid as well. `@` is the epoch
|
||||
/// and the statement starts 5e18 ms later, so the fold offset is that same 5e18 ms and the sum the
|
||||
/// horizon reads is 1e19 ms - not an instant an `i64` holds. The runtime refuses the window while
|
||||
/// it builds it.
|
||||
///
|
||||
/// The window is wrapped in `topk`, a whole-vector consumer: it makes the route cut the window as a
|
||||
/// segment and decide the statement's plan from that segment while planning, which is the moment
|
||||
/// the refusal cannot be left to the executor - a native plan that fails while its physical plan is
|
||||
/// built has none to fall back to. Deciding this during planning is what the route did while it
|
||||
/// decided admission by building the runtime and dropping it, and it is what the admission
|
||||
/// predicate decides now: every site that commits to a tree asks `native_tree_supported` and builds
|
||||
/// no runtime, so `window_instants_fit` is the only thing that keeps this query off the native
|
||||
/// route, and dropping it from the predicate makes this test fail. (The `@` half of the same
|
||||
/// arithmetic is pinned one level down, in `native::plan::tests`, because a statement whose anchor
|
||||
/// and start are that far apart is refused by the planner's own at-modifier rewrite first.)
|
||||
#[tokio::test]
|
||||
async fn native_fused_windows_whose_instants_overflow_fall_back() {
|
||||
// The far end of the `i64` millisecond range, one step apart.
|
||||
let plan_for = |query: &str, native_fused| {
|
||||
let query = query.to_string();
|
||||
async move {
|
||||
let mut eval_stmt = build_eval_stmt(&query);
|
||||
eval_stmt.start = UNIX_EPOCH
|
||||
.checked_add(Duration::from_millis(5_000_000_000_000_000_000))
|
||||
.unwrap();
|
||||
eval_stmt.end = eval_stmt
|
||||
.start
|
||||
.checked_add(Duration::from_millis(10_000))
|
||||
.unwrap();
|
||||
let provider = build_test_table_provider_with_options(
|
||||
&[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())],
|
||||
1,
|
||||
1,
|
||||
vec![],
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
ConcreteDataType::float64_datatype(),
|
||||
NATIVE_ENGINE,
|
||||
)
|
||||
.await;
|
||||
PromPlanner::stmt_to_plan_with_annotations(
|
||||
provider,
|
||||
&eval_stmt,
|
||||
&build_query_engine_state(),
|
||||
None,
|
||||
native_fused,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
// `@ 0` folds to the grid's own start, which the horizon then adds to a point of the grid.
|
||||
let query = "topk(1, predict_linear(some_metric[1m] @ 0, 60))";
|
||||
let native = plan_for(query, true).await;
|
||||
let fallback = plan_for(query, false).await;
|
||||
assert!(
|
||||
find_fused_series(&native).is_none(),
|
||||
"{query} reads an instant the runtime cannot compute and must keep the existing plan: {}",
|
||||
native.display_indent()
|
||||
);
|
||||
assert_eq!(
|
||||
native.display_indent().to_string(),
|
||||
fallback.display_indent().to_string(),
|
||||
"{query}: the native setting must plan exactly what the existing planner plans"
|
||||
);
|
||||
// The same shape over a statement near the epoch fits, so the route is not narrowed.
|
||||
let near = plan_for("topk(1, rate(some_metric[5m]))", true).await;
|
||||
assert!(
|
||||
find_fused_series(&near).is_some(),
|
||||
"a window that fits must keep the native route: {}",
|
||||
near.display_indent()
|
||||
);
|
||||
}
|
||||
|
||||
/// A collapsing rewrite on an unsplit table lowers the same canonical tree to its producer.
|
||||
#[tokio::test]
|
||||
async fn native_fused_label_rewrite_lowers_canonical_boundary() {
|
||||
|
||||
Reference in New Issue
Block a user