diff --git a/src/promql/src/map.rs b/src/promql/src/map.rs index 2d2c4b80a7b..cb3c3cc6f80 100644 --- a/src/promql/src/map.rs +++ b/src/promql/src/map.rs @@ -318,7 +318,7 @@ fn go_min(x: f64, y: f64) -> f64 { impl ArithmeticOp { /// Prometheus `vectorElemBinop` for two floats. - pub(crate) fn calculate(self, left: f64, right: f64) -> f64 { + pub fn calculate(self, left: f64, right: f64) -> f64 { match self { Self::Add => left + right, Self::Subtract => left - right, diff --git a/src/query/src/promql/native_compile.rs b/src/query/src/promql/native_compile.rs index 1be936d7980..56ea7445b76 100644 --- a/src/query/src/promql/native_compile.rs +++ b/src/query/src/promql/native_compile.rs @@ -16,20 +16,31 @@ //! //! This is the front half of the native evaluation model: it reads the parsed statement and writes //! down what it means as bound nodes. It runs nothing, and nothing calls it yet, so the subset it -//! accepts is deliberately three shapes: +//! accepts is one recursive rule: //! //! ```text -//! e.g. x -//! ([]) e.g. rate(x[5m]) -//! (, []) e.g. quantile_over_time(0.9, x[5m]) +//! e.g. x +//! ([]) e.g. rate(x[5m]) +//! (, []) e.g. quantile_over_time(0.9, x[5m]) +//! ([, ]*) e.g. abs(rate(x[5m])), clamp_min(x, 0) //! ``` //! -//! which become `Input` (raw samples) → `Instant` (the last sample within the statement's lookback) -//! and `Input` (raw samples) → `Window` (the range function over the statement's grid). A selector -//! may carry `offset` in either shape and, in the window shape, an `@` anchor. Everything else is a -//! [`DataFusionError::NotImplemented`] that names what was found and quotes the shapes above, -//! because rejecting a query the compiler does not understand is the only honest alternative to -//! compiling it into something else. +//! which become `Input` (raw samples) → `Instant` (the last sample within the statement's lookback), +//! `Input` (raw samples) → `Window` (the range function over the statement's grid) and +//! `` → `Map` (a pointwise math function over what its argument evaluates to). A +//! selector may carry `offset` in either selection shape and, in the window shape, an `@` anchor. +//! Everything else is a [`DataFusionError::NotImplemented`] that names what was found and quotes the +//! shapes above, because rejecting a query the compiler does not understand is the only honest +//! alternative to compiling it into something else. +//! +//! Two decisions keep the recursion from growing a branch per query. A constant argument — a +//! quantile, a rounding step, a clamp bound — is folded out of the statement by [`constant_value`], +//! so `quantile_over_time(1 - 0.1, x[5m])` binds the number it means rather than the expression it +//! was written as. A math function is bound by [`math_op`] from a list of names, so +//! `clamp_min(abs(rate(x[5m])), 0)` is two maps over one window because the arguments nest, not +//! because the compiler knows the query. Arithmetic inside a fold is [`ArithmeticOp::calculate`]'s +//! alone, so the number this compiler binds and the number the kernel computes cannot disagree +//! about `%`, `^` or `atan2`. //! //! The statement's grid, the selector's lookback, window, offset and anchor are the only numbers //! derived here; the legality of the tree is not checked here. [`NativeTree::try_new`] is the @@ -45,25 +56,28 @@ use datafusion::error::{DataFusionError, Result}; use datatypes::series::SeriesOrdering; use datatypes::series::samples::SeriesSchema; use datatypes::series::time::{EvalGrid, GridSpec}; +use promql::map::{ArithmeticOp, MathOp}; use promql::native::plan::{ - BoundInput, BoundInstant, BoundWindow, NativeExpr, NativeNode, NativeTree, NativeType, + BoundInput, BoundInstant, BoundMap, BoundWindow, NativeExpr, NativeNode, NativeTree, NativeType, }; use promql::range_eval::{RangeFunction, RangeWindow}; +use promql_parser::parser::token::TokenType; +use promql_parser::parser::value::ValueType; use promql_parser::parser::{ AtModifier, Call, EvalStmt, Expr as PromExpr, MatrixSelector, NumberLiteral, Offset, - VectorSelector, + VectorSelector, token, }; use crate::promql::label_values::signed_millis_since_epoch; /// The shapes this compiler accepts, quoted in every rejection. -const SUPPORTED: &str = "a bare selector, a range function over a selector — \ - `([])` — or a parameterised one, \ - `(, [])`, for example `x`, \ - `rate(x[5m])` or `quantile_over_time(0.9, x[5m])`"; +const SUPPORTED: &str = "a selector, a range function over a range selector — \ + `([])`, with its constants in front of \ + the selector as in `quantile_over_time(0.9, x[5m])` — or a math function \ + over any of them, as in `abs(rate(x[5m]))`: for example `x`, \ + `rate(x[5m])`, `quantile_over_time(0.9, x[5m])` or `abs(rate(x[5m]))`"; -/// Compiles `stmt` into a tree that reads `input` and either selects from it or folds it with the -/// statement's one window. +/// Compiles `stmt` into a tree that reads `input` and evaluates the statement's expression over it. /// /// `input` is the raw series schema the caller bound from the table, and the caller has to apply the /// selector's matchers before binding it: `__field__`, `__schema__` and `__database__` select rows @@ -92,8 +106,7 @@ pub fn compile( ) -> Result { // Every shape this compiler accepts reads one series to its last chunk before the next, so raw // samples whose chunks of different series interleave cannot be fed to any of them. The refusal - // is here rather than in a shape's branch so that a shape added later inherits it, and it comes - // before the early return below for a bare selector. + // is here rather than in a shape's branch so that a shape added later inherits it. if ordering == SeriesOrdering::Interleaved { return Err(not_implemented( "raw samples whose series interleave, which this compiler cannot consume: a native \ @@ -101,47 +114,137 @@ pub fn compile( series may not interleave", )); } - // A bare selector is the other shape: an instant selection over the same raw input. - if let PromExpr::VectorSelector(selector) = unparen(&stmt.expr) { - return instant_selection(stmt, selector, input, ordering); - } - let PromExpr::Call(Call { func, args }) = unparen(&stmt.expr) else { - return Err(not_implemented(&format!("the expression `{}`", stmt.expr))); + let context = Context { + stmt, + input, + ordering, }; - let Some(signature) = range_signature(func.name) else { + // The whole tree is validated once, here: a node compiled as another node's input is checked as + // part of the tree rather than on its own, so the same node cannot be legal in one place and + // illegal in another. + NativeTree::try_new(compile_expr(&context, &stmt.expr)?) +} + +/// What every node of one compilation derives from the statement rather than from the expression. +/// +/// The three are the same for the whole tree: one statement decides the grid, the lookback and the +/// anchors, one caller binds the raw input, and one scan guarantees the ordering. Passing them as +/// one value is what lets [`compile_expr`] recurse without restating what it was handed. +struct Context<'a> { + /// The statement being compiled. + stmt: &'a EvalStmt, + /// The raw series the caller bound; shared by every input node rather than copied per node. + input: Arc, + /// The ordering the caller's scan guarantees. + ordering: SeriesOrdering, +} + +/// Compiles `expr` into the node it denotes, recursing into whatever it is made of. +/// +/// One rule per kind of expression and nothing per query: a math function is compiled by compiling +/// its argument, so a query nests as deeply as it was written. A kind this compiler has no node for +/// is refused with what was found, which is why the rejection is useful without a coverage list. +fn compile_expr(context: &Context, expr: &PromExpr) -> Result { + match expr { + // Parentheses group, so they change how a statement was typed rather than what it asks for: + // the expression inside them is the one that compiles. + PromExpr::Paren(paren) => compile_expr(context, &paren.expr), + PromExpr::Call(call) => compile_call(context, call), + PromExpr::VectorSelector(selector) => instant_selection(context, selector), + found => Err(not_implemented(&format!("the expression `{found}`"))), + } +} + +/// Compiles a call: a range function opens a window over the selector it is given, and a math +/// function maps the value of its first argument. +/// +/// The two name lists are disjoint, and a range function is decided first because it binds its own +/// window rather than its argument's value. A name in neither list is refused here, which is where +/// the difference between "this compiler cannot fold this function" and "this name means nothing" +/// is still visible. +fn compile_call(context: &Context, call: &Call) -> Result { + let Call { func, args } = call; + if let Some(signature) = range_signature(func.name) { + return window_call(context, func.name, signature, &args.args); + } + // The name is looked up before the call's arguments are read, so a name this compiler has no + // kernel for is refused as such whichever arguments it was given: `pi()` and `time()` have no + // vector to map, and the rejection names the name rather than describing a vector they never + // wrote. + let Some((op, vector)) = math_op(func.name, &args.args)? else { return Err(not_implemented(&format!( - "a call to `{}`, which has no native range kernel", + "a call to `{}`, which has no native kernel", func.name ))); }; + map_call(context, func.name, op, vector) +} + +/// Binds a range function over the range selector it is given, and the window that selector opens. +/// +/// The parameters come first and the range selector is last, for both signatures: a function without +/// parameters has no `parameters` to read. +fn window_call( + context: &Context, + name: &str, + signature: RangeSignature, + args: &[Box], +) -> Result { // The count is the signature's, so the rejection names what this function takes rather than // what one argument happens to be. let expected = signature.parameters() + 1; - if args.args.len() != expected { + if args.len() != expected { return Err(not_implemented(&format!( - "a call to `{}` with {} argument(s), where it takes {expected}: {}", - func.name, - args.args.len(), + "a call to `{name}` with {} argument(s), where it takes {expected}: {}", + args.len(), signature.arguments() ))); } - // The parameters come first and the range selector is last, for both signatures: a function - // without parameters has no `parameters` to read. - let (parameters, selector) = args.args.split_at(signature.parameters()); + let (parameters, selector) = args.split_at(signature.parameters()); let function = match signature { RangeSignature::Selector(function) => function, - RangeSignature::Scalar(bind) => bind(scalar_parameter(func.name, ¶meters[0])?), + RangeSignature::Scalar(bind) => bind(constant_argument(name, ¶meters[0], 0)?), }; let argument = &selector[0]; let PromExpr::MatrixSelector(MatrixSelector { vs, range }) = unparen(argument) else { return Err(not_implemented(&format!( - "a call to `{}` whose argument is `{}`", - func.name, argument + "a call to `{name}` whose argument is `{argument}`" ))); }; - // The ordering was refused at the entry above, so every shape below is built over a scan whose - // series' chunks do not interleave. - window_selection(stmt, function, vs, range, input, ordering) + window_selection(context, function, vs, range) +} + +/// Wraps `op` over the value of `vector`, which is the first argument of a math function. +/// +/// The output is the input's own type: a pointwise function moves no value to another grid point +/// and reports one value per field it was given, so the map's schema is the input's allocation +/// rather than a copy that merely agrees with it. [`NativeTree::validate`] does not require that +/// sharing: it compares the map's grid with its input's by [`EvalGrid::spec`] and its field count +/// with its input's, so a map that allocated an equal grid of its own would validate the same way. +/// The only comparison by identity is an input node's, whose output has to be the very binding it +/// was bound to. +fn map_call(context: &Context, name: &str, op: MathOp, vector: &PromExpr) -> Result { + // A math function is pointwise over a vector, so its first argument has to be one. The parser + // checks the same thing and the tree's validation checks the compiled input's type again, but + // neither can name the argument the client wrote: this is the check that can. + if vector.value_type() != ValueType::Vector { + return Err(not_implemented(&format!( + "a call to `{name}` whose first argument is `{vector}`, which is a {}, where the \ + function takes a vector", + vector.value_type() + ))); + } + let input = compile_expr(context, vector)?; + let output = input.output.clone(); + Ok(NativeNode { + expr: NativeExpr::Map { + input: Box::new(input), + op: BoundMap { + ops: Arc::from([op]), + }, + }, + output, + }) } /// Binds `function` over the range selector `vs[range]` into a window over the raw input. @@ -150,39 +253,37 @@ pub fn compile( /// whichever function reads them, so a parameterised function only decides *what* folds the window, /// never *which* window is folded. fn window_selection( - stmt: &EvalStmt, + context: &Context, function: RangeFunction, vs: &VectorSelector, range: &Duration, - input: Arc, - ordering: SeriesOrdering, -) -> Result { +) -> Result { // An `@` anchor is step-invariant: every step folds the same samples, which the window's // `anchor_ms` tells `RangeEvaluator` to replay over the grid. let range_ms = range_millis(range.as_millis())?; let offset_ms = selector_offset_ms(&vs.offset)?; - let anchor_ms = selector_anchor_ms(&vs.at, stmt)?; + let anchor_ms = selector_anchor_ms(&vs.at, context.stmt)?; // The grid is the last thing built: a query refused above, or a window that cannot exist, // never pays for a grid nothing would read. - let output_grid = statement_grid(stmt)?; + let output_grid = statement_grid(context.stmt)?; // A range function reports one value per input field, so the layout is the input's. let output = NativeType::Samples(Arc::new(SeriesSchema { - fields: Arc::clone(&input.fields), + fields: Arc::clone(&context.input.fields), eval_grid: Some(Arc::clone(&output_grid)), })); // One allocation shared by the binding and the node's output: `NativeTree::try_new` compares // them by identity, because a compiler that meant one schema has no reason to allocate it // twice. - let raw = NativeType::Samples(input); + let raw = NativeType::Samples(Arc::clone(&context.input)); - NativeTree::try_new(NativeNode { + Ok(NativeNode { expr: NativeExpr::Window { input: Box::new(NativeNode { expr: NativeExpr::Input(BoundInput { output: raw.clone(), - ordering, + ordering: context.ordering, }), output: raw, }), @@ -206,12 +307,7 @@ fn window_selection( /// /// The lookback, the offset and the grid are the whole of an instant selection: it reports the last /// sample at or before each grid point, within the statement's lookback. -fn instant_selection( - stmt: &EvalStmt, - selector: &VectorSelector, - input: Arc, - ordering: SeriesOrdering, -) -> Result { +fn instant_selection(context: &Context, selector: &VectorSelector) -> Result { // An `@` anchor is bound on a range window, which carries `anchor_ms`; an instant selection has // no such field, so the anchor has nowhere to go. Refusing it is the only alternative to // dropping it, which would silently select around every grid point instead of one anchor. @@ -222,25 +318,25 @@ fn instant_selection( anchoring is bound on a range window, as in `rate(x[5m] @ 1234)`" ))); } - let lookback_ms = lookback_millis(stmt.lookback_delta.as_millis())?; + let lookback_ms = lookback_millis(context.stmt.lookback_delta.as_millis())?; let offset_ms = selector_offset_ms(&selector.offset)?; - let output_grid = statement_grid(stmt)?; + let output_grid = statement_grid(context.stmt)?; // An instant selection reports one value per input field, so the layout is the input's. let output = NativeType::Samples(Arc::new(SeriesSchema { - fields: Arc::clone(&input.fields), + fields: Arc::clone(&context.input.fields), eval_grid: Some(Arc::clone(&output_grid)), })); // One allocation shared by the binding and the node's output, as in the window shape. - let raw = NativeType::Samples(input); + let raw = NativeType::Samples(Arc::clone(&context.input)); - NativeTree::try_new(NativeNode { + Ok(NativeNode { expr: NativeExpr::Instant { input: Box::new(NativeNode { expr: NativeExpr::Input(BoundInput { output: raw.clone(), - ordering, + ordering: context.ordering, }), output: raw, }), @@ -356,10 +452,10 @@ fn grid_error(error: datatypes::error::Error) -> DataFusionError { enum RangeSignature { /// A function that reads only its range selector, as in `rate(x[5m])`. Selector(RangeFunction), - /// A function that reads one number literal before its range selector, and the function that - /// number binds, as in `quantile_over_time(0.9, x[5m])`. + /// A function that reads one constant before its range selector, and the function that constant + /// binds, as in `quantile_over_time(0.9, x[5m])`. /// - /// The number is kept out of the variant name because only the call site knows it. + /// The constant is kept out of the variant name because only the call site knows it. Scalar(fn(f64) -> RangeFunction), } @@ -381,7 +477,7 @@ impl RangeSignature { } } -/// The signature the PromQL name calls, or `None` when this compiler has no kernel for it. +/// The signature the PromQL name calls, or `None` when this compiler has no range kernel for it. /// /// The map is explicit on purpose, as in [`range_function`]: a name that is not listed is rejected, /// so supporting one more function is a decision made in one place rather than a consequence of a @@ -394,29 +490,235 @@ fn range_signature(name: &str) -> Option { } } -/// The number a parameterised range function was given, as the literal it was written as. +/// What a call to a math function has to be: the constants it reads after its vector, and how to +/// bind them. /// -/// A number literal is the only parameter this compiler binds. A parameter whose value is only -/// known at run time, `time()` being the real case, would make the kernel's constant a query-time -/// value, which the tree has no field for. +/// The shapes are separated by how many constants the function reads, because that is the only +/// thing the variants differ in: a name maps to one kernel, and the constants are bound from the +/// call. `round` is the one function whose constant may be left out, so it is its own shape: the +/// default is a decision about the function rather than about the constant. +#[derive(Clone, Copy)] +enum MathSignature { + /// A function of its vector alone, as in `abs`. + Unary(MathOp), + /// A function of its vector and one constant, as in `clamp_min`. + OneConstant(fn(f64) -> MathOp), + /// A function of its vector and one constant it may be called without, as in `round`. + DefaultedConstant { + default: f64, + bind: fn(f64) -> MathOp, + }, + /// A function of its vector and two constants, as in `clamp`. + TwoConstants(fn(f64, f64) -> MathOp), +} + +impl MathSignature { + /// How many constants the function reads when every one of them is written. + fn parameters(self) -> usize { + match self { + Self::Unary(_) => 0, + Self::OneConstant(_) | Self::DefaultedConstant { .. } => 1, + Self::TwoConstants(_) => 2, + } + } + + /// How many constants the function has to be given. + fn required(self) -> usize { + match self { + Self::DefaultedConstant { .. } => 0, + _ => self.parameters(), + } + } + + /// What the call's arguments are, for the arity rejection to name them. + fn arguments(self) -> &'static str { + match self { + Self::Unary(_) => "its vector", + Self::OneConstant(_) => "its vector and one constant", + Self::DefaultedConstant { .. } => "its vector and one optional constant", + Self::TwoConstants(_) => "its vector and two constants", + } + } +} + +/// The signature the PromQL name calls, or `None` when this compiler has no math kernel for it. /// -/// A parameter written as an expression the existing planner folds to a literal is refused here -/// and is a gap this stage leaves open on purpose: `quantile_over_time(1 - 0.1, x[5m])` and -/// `quantile_over_time(-(0.9), x[5m])` plan today through `try_build_literal_expr`, which folds -/// `Paren`, `Unary` and literal-only `Binary`, and reach this function still unfolded. They are -/// refused rather than answered from the wrong number, and folding them here would be the work -/// to close the gap. +/// The map is explicit on purpose, as in [`range_function`]: a name that is not listed is refused +/// rather than treated as math because it looks mathematical, so adding one more function is a +/// decision made in one place. A name in this list is a promise that its kernel exists, which is why +/// arity is a property of the variant rather than of the caller. +fn math_signature(name: &str) -> Option { + match name { + "abs" => Some(MathSignature::Unary(MathOp::Abs)), + "ceil" => Some(MathSignature::Unary(MathOp::Ceil)), + "floor" => Some(MathSignature::Unary(MathOp::Floor)), + "sqrt" => Some(MathSignature::Unary(MathOp::Sqrt)), + "exp" => Some(MathSignature::Unary(MathOp::Exp)), + "ln" => Some(MathSignature::Unary(MathOp::Ln)), + "log2" => Some(MathSignature::Unary(MathOp::Log2)), + "log10" => Some(MathSignature::Unary(MathOp::Log10)), + "sin" => Some(MathSignature::Unary(MathOp::Sin)), + "cos" => Some(MathSignature::Unary(MathOp::Cos)), + "tan" => Some(MathSignature::Unary(MathOp::Tan)), + "asin" => Some(MathSignature::Unary(MathOp::Asin)), + "acos" => Some(MathSignature::Unary(MathOp::Acos)), + "atan" => Some(MathSignature::Unary(MathOp::Atan)), + "sinh" => Some(MathSignature::Unary(MathOp::Sinh)), + "cosh" => Some(MathSignature::Unary(MathOp::Cosh)), + "tanh" => Some(MathSignature::Unary(MathOp::Tanh)), + "asinh" => Some(MathSignature::Unary(MathOp::Asinh)), + "acosh" => Some(MathSignature::Unary(MathOp::Acosh)), + "atanh" => Some(MathSignature::Unary(MathOp::Atanh)), + "deg" => Some(MathSignature::Unary(MathOp::Deg)), + "rad" => Some(MathSignature::Unary(MathOp::Rad)), + "sgn" => Some(MathSignature::Unary(MathOp::Sgn)), + // `round`'s nearest defaults to one, so `round(x)` is `round(x, 1)` and its variant always + // carries the number it rounds to. + "round" => Some(MathSignature::DefaultedConstant { + default: 1.0, + bind: |nearest| MathOp::Round { nearest }, + }), + "clamp" => Some(MathSignature::TwoConstants(|min, max| MathOp::Clamp { + min, + max, + })), + "clamp_min" => Some(MathSignature::OneConstant(|min| MathOp::ClampMin { min })), + "clamp_max" => Some(MathSignature::OneConstant(|max| MathOp::ClampMax { max })), + _ => None, + } +} + +/// The pointwise kernel `name` denotes over its vector, and that vector, or `None` when this +/// compiler has no math kernel for that name. +/// +/// A name that is not in [`math_signature`] is not math: the caller reports that it has no kernel +/// instead of compiling it into a kernel that would compute something else. `arguments` is the +/// call's whole argument list, because the vector is its first argument: the arity is the +/// signature's and counts the vector in, so a call with no argument at all is refused as a call +/// that wrote none rather than as a shape this compiler cannot bind. The arguments after the vector +/// are the call's constants, in the order they were written, and each is folded to the number it +/// means by [`constant_value`], because the tree binds a kernel's constants and has no field for an +/// expression. +fn math_op<'a>( + name: &str, + arguments: &'a [Box], +) -> Result> { + let Some(signature) = math_signature(name) else { + return Ok(None); + }; + // The vector is the function's first argument, so a call that wrote no argument at all is + // refused by the same arity check as one that wrote too few constants. + let Some((vector, parameters)) = arguments.split_first() else { + return Err(arity_error(name, signature, 0)); + }; + let given = parameters.len() + 1; + if given < signature.required() + 1 || given > signature.parameters() + 1 { + return Err(arity_error(name, signature, given)); + } + // The vector is the first argument, so the constants are read from the second onwards. + let op = match signature { + MathSignature::Unary(op) => op, + MathSignature::OneConstant(bind) => bind(constant_argument(name, ¶meters[0], 1)?), + MathSignature::DefaultedConstant { default, bind } => bind(match parameters.first() { + Some(parameter) => constant_argument(name, parameter, 1)?, + None => default, + }), + MathSignature::TwoConstants(bind) => bind( + constant_argument(name, ¶meters[0], 1)?, + constant_argument(name, ¶meters[1], 2)?, + ), + }; + Ok(Some((op, vector))) +} + +/// The rejection of a math call that wrote `given` arguments where `signature` takes another count. +/// +/// The count is the signature's, so the message says what this function takes rather than what one +/// argument happens to be. The vector is counted in: it is the function's first argument, and a +/// reader counting the call's arguments should not have to add it themselves. +fn arity_error(name: &str, signature: MathSignature, given: usize) -> DataFusionError { + not_implemented(&format!( + "a call to `{name}` with {given} argument(s), where it takes {}", + signature.arguments() + )) +} + +/// The constant `argument` folds to, or a rejection naming the argument and where it was read. +/// +/// A parameter is a constant of the statement: a quantile, a rounding step or a clamp bound is a +/// field of the tree, so a parameter whose value is only known at run time — `time()` being the real +/// case — has nowhere to go. It is refused rather than answered from the wrong number, and the +/// rejection names the expression, because that is what the query has to change. /// /// A quantile outside `[0, 1]` is not refused here: Prometheus binds it and the kernel reports -/// `-Infinity` below zero and `+Infinity` above one, so refusing it would be this compiler -/// inventing a rule the query engine does not have. -fn scalar_parameter(name: &str, parameter: &PromExpr) -> Result { - match unparen(parameter) { - PromExpr::NumberLiteral(NumberLiteral { val }) => Ok(*val), - found => Err(not_implemented(&format!( - "a call to `{name}` whose first argument is `{found}`, where it has to be a number \ - literal" - ))), +/// `-Infinity` below zero and `+Infinity` above one, so refusing it would be this compiler inventing +/// a rule the query engine does not have. +fn constant_argument(name: &str, argument: &PromExpr, position: usize) -> Result { + constant_value(argument).ok_or_else(|| { + not_implemented(&format!( + "a call to `{name}` whose {} is `{argument}`, where it has to be a constant expression \ + this compiler folds to a number literal", + argument_position(position) + )) + }) +} + +/// How a rejection names the argument a parameter was read from. +/// +/// The three positions are the ones a signature can read: the vector, and the two constants behind +/// it. Every caller passes the literal position its signature reads, so there is no fourth position +/// to name and no default to invent one for. +fn argument_position(position: usize) -> &'static str { + const POSITIONS: [&str; 3] = ["first argument", "second argument", "third argument"]; + POSITIONS[position] +} + +/// The number `expr` folds to, or `None` when its value is not the statement's to know. +/// +/// Only what the statement itself decides is folded: a number literal, a group around one, a negated +/// one, and arithmetic whose two operands both fold. A call, a selector, a range selector, an +/// aggregate and a subquery are `None`, because their value comes from the data or from the +/// evaluation time, and a kernel's constant is a field of the tree rather than something bound while +/// it runs. Returning `None` rather than an error is what lets the caller report where the +/// expression was found, which is the useful half of the refusal. +/// +/// Arithmetic is [`ArithmeticOp::calculate`]'s and nothing else's: that is where Prometheus' +/// semantics for `%`, `^` and `atan2` live, the last two through `go_math`, so defining them again +/// here would be a second answer for the two to disagree over. An operator with no +/// [`ArithmeticOp`] — a comparison, `and`, `or`, `unless` — is not arithmetic and does not fold. +fn constant_value(expr: &PromExpr) -> Option { + match expr { + PromExpr::NumberLiteral(NumberLiteral { val }) => Some(*val), + PromExpr::Paren(paren) => constant_value(&paren.expr), + PromExpr::Unary(unary) => constant_value(&unary.expr).map(|value| -value), + PromExpr::Binary(binary) => { + let op = arithmetic_op(&binary.op)?; + let left = constant_value(&binary.lhs)?; + let right = constant_value(&binary.rhs)?; + Some(op.calculate(left, right)) + } + // Calls (`time()`, `scalar(v)`), selectors, range selectors, aggregates and subqueries all + // read something this compiler cannot know before the query runs. + _ => None, + } +} + +/// The arithmetic operator a PromQL token denotes, or `None` when the token is not arithmetic. +/// +/// The token is matched by its id, which is stable and is what the rest of the planner matches on, +/// rather than by the text it displays as: a symbol this compiler does not recognise is not +/// arithmetic here rather than a wrong operator. A comparison and a set operator are the cases that +/// reach this and fold nothing, which is this batch's boundary rather than an accident of the map. +fn arithmetic_op(op: &TokenType) -> Option { + match op.id() { + token::T_ADD => Some(ArithmeticOp::Add), + token::T_SUB => Some(ArithmeticOp::Subtract), + token::T_MUL => Some(ArithmeticOp::Multiply), + token::T_DIV => Some(ArithmeticOp::Divide), + token::T_MOD => Some(ArithmeticOp::Modulo), + token::T_POW => Some(ArithmeticOp::Power), + token::T_ATAN2 => Some(ArithmeticOp::Atan2), + _ => None, } } @@ -452,7 +754,8 @@ fn range_function(name: &str) -> Option { /// /// Parentheses group, so they say nothing about the expression inside them: `rate((x[5m]))` asks /// for exactly what `rate(x[5m])` asks for. Refusing it would reject a statement on the way it was -/// typed rather than on what it means. +/// typed rather than on what it means. A range selector is not an expression this compiler +/// compiles on its own, so the call that reads one looks through the grouping itself. fn unparen(mut expr: &PromExpr) -> &PromExpr { while let PromExpr::Paren(paren) = expr { expr = &paren.expr; diff --git a/src/query/src/promql/native_compile/tests.rs b/src/query/src/promql/native_compile/tests.rs index 05b4d07ead8..f16bb74b072 100644 --- a/src/query/src/promql/native_compile/tests.rs +++ b/src/query/src/promql/native_compile/tests.rs @@ -18,8 +18,9 @@ use std::time::{Duration, UNIX_EPOCH}; use datafusion::error::DataFusionError; use datatypes::series::SeriesOrdering; use datatypes::series::samples::{FieldSpec, SeriesSchema}; +use promql::map::MathOp; use promql::native::plan::{ - BoundInstant, BoundWindow, NativeExpr, NativeNode, NativeTree, NativeType, + BoundInstant, BoundMap, BoundWindow, NativeExpr, NativeNode, NativeTree, NativeType, }; use promql::range_eval::RangeFunction; use promql_parser::parser::{EvalStmt, Expr as PromExpr, NumberLiteral}; @@ -82,6 +83,18 @@ fn with_call_arguments(query: &str, edit: impl FnOnce(&mut Vec>)) stmt } +/// The range selector out of `rate(x[5m])`. +/// +/// The parser type-checks function arguments, so a range selector can only be handed to a function +/// that does not take one by editing a tree the parser accepted. +fn a_matrix_selector() -> Box { + let mut stmt = eval_stmt("rate(x[5m])"); + let PromExpr::Call(call) = &mut stmt.expr else { + unreachable!("`rate(x[5m])` parses to a call"); + }; + call.args.args.remove(0) +} + /// The window at the root of `tree`, for the range-function shape. fn root_window(tree: &NativeTree) -> (&NativeNode, &BoundWindow) { let NativeExpr::Window { input, op } = &tree.root().expr else { @@ -101,6 +114,32 @@ fn root_instant(tree: &NativeTree) -> (&NativeNode, &BoundInstant) { (input, op) } +/// The input and the operation of a `Map` node. +fn map_parts(node: &NativeNode) -> (&NativeNode, &BoundMap) { + let NativeExpr::Map { input, op } = &node.expr else { + panic!("the node is not a map: {:?}", node.expr); + }; + (input, op) +} + +/// The one operation a `Map` node applies. +fn map_op(node: &NativeNode) -> MathOp { + let (_, op) = map_parts(node); + let [only] = &op.ops[..] else { + panic!("the map holds {} operation(s)", op.ops.len()); + }; + *only +} + +/// The schema a node's evaluated output carries. +fn evaluated(node: &NativeNode) -> &Arc { + let Some(schema) = node.output.samples() else { + panic!("the node does not produce samples: {:?}", node.output); + }; + assert!(schema.eval_grid.is_some(), "the node is not evaluated"); + schema +} + /// `rate(x[5m])` becomes a window over the raw input, on the statement's grid. #[test] fn a_rate_over_a_selector_becomes_a_window_over_raw_input() { @@ -277,27 +316,29 @@ fn a_parenthesised_selector_binds_as_written() { } } -/// A parameter the planner folds to a literal is still refused, and that gap is deliberate: the -/// kernel's constant is a field of the tree, so binding `1 - 0.1` means folding it here rather than -/// carrying an expression. A refusal is the agreed behaviour for this stage; an answer computed -/// from the wrong number would not be. +/// A parameter written as an expression binds the number it folds to, because the kernel's constant +/// is a field of the tree: `1 - 0.1` is the quantile `0.9` written two ways, and carrying the +/// expression would bind nothing at all. The value is asserted exactly, so a fold that produced a +/// neighbouring number rather than the arithmetic's own result would fail here. #[test] -fn a_parameter_folded_from_an_expression_is_refused() { - for query in [ - "quantile_over_time(1 - 0.1, x[5m])", - "quantile_over_time(-(0.9), x[5m])", - ] { - let error = compile(&eval_stmt(query), raw_schema(), SeriesOrdering::KeyThenTime) - .expect_err(&format!("`{query}` compiled, and it should not have")); - assert!( - matches!(error, DataFusionError::NotImplemented(_)), - "`{query}` did not report NotImplemented: {error}" - ); - let message = error.to_string(); - assert!( - message.contains("number literal"), - "`{query}` did not say what the argument has to be: {message}" +fn a_parameter_folded_from_an_expression_binds_its_value() { + // `1 - 0.1` is `0.9` bit for bit in IEEE754, and `-(0.9)` is the negative of the same number: + // negation is exact, and the fold does not decide whether a quantile is inside `[0, 1]`. + let cases = [ + ("quantile_over_time(1 - 0.1, x[5m])", 0.9), + ("quantile_over_time(-(0.9), x[5m])", -0.9), + ]; + + for (query, quantile) in cases { + let tree = compile(&eval_stmt(query), raw_schema(), SeriesOrdering::KeyThenTime) + .unwrap_or_else(|error| panic!("`{query}` was refused: {error}")); + let (_, op) = root_window(&tree); + assert_eq!( + op.function, + RangeFunction::QuantileOverTime(quantile), + "`{query}` did not bind the quantile it folds to" ); + assert_eq!(op.window.range_ms, 300_000, "`{query}` lost its window"); } } @@ -369,6 +410,23 @@ fn a_call_with_the_wrong_number_of_arguments_names_the_count_it_takes() { "rate", "where it takes 1", ), + // A math function's constants are counted the same way: `clamp` reads two of them after its + // vector, so one is one too few. + ( + with_call_arguments("clamp(x, 0, 1)", |args| { + args.pop(); + }), + "clamp", + "where it takes its vector and two constants", + ), + // A listed math name called with no argument at all: the vector is counted as the first + // argument, so a call that wrote none is one argument short of `abs` rather than a shape + // the compiler has no words for. + ( + with_call_arguments("abs(x)", |args| args.clear()), + "abs", + "with 0 argument(s), where it takes its vector", + ), ]; for (stmt, name, expected) in cases { @@ -713,9 +771,9 @@ fn the_numeric_edges_of_a_statement_are_bound_as_given() { #[test] fn an_interleaved_scan_is_refused_whatever_the_shape() { // The plain range function is the ordering table's case, which compiles `rate(x[5m])` under - // `Interleaved` and expects the same refusal; these two cover the shapes that table cannot, - // because one returns before a window exists and the other takes a parameter. - for query in ["x", "quantile_over_time(0.9, x[5m])"] { + // `Interleaved` and expects the same refusal; these cover the shapes that table cannot: the + // bare selector, a math function over one, and a parameterised range function. + for query in ["x", "abs(x)", "quantile_over_time(0.9, x[5m])"] { let error = compile(&eval_stmt(query), raw_schema(), SeriesOrdering::Interleaved) .expect_err(&format!("`{query}` compiled over an interleaved scan")); assert!( @@ -765,12 +823,16 @@ fn an_unsupported_shape_is_rejected_and_names_what_it_found() { // A function with no kernel, whether or not it takes a constant argument. ( "predict_linear(x[5m], 300)", - "a call to `predict_linear`, which has no native range kernel", + "a call to `predict_linear`, which has no native kernel", ), ( "absent(x)", - "a call to `absent`, which has no native range kernel", + "a call to `absent`, which has no native kernel", ), + // The two functions PromQL calls with no arguments at all: neither has a kernel, and the + // rejection names the name rather than describing a vector they were never given. + ("pi()", "a call to `pi`, which has no native kernel"), + ("time()", "a call to `time`, which has no native kernel"), ]; for (query, named) in cases { @@ -840,3 +902,260 @@ fn parentheses_around_the_accepted_shape_are_transparent() { ); } } +/// A math function compiles by compiling the expression it is applied to, so its node sits over the +/// node that expression produces: `abs(rate(x[5m]))` is a map over the window `rate(x[5m])` is, not +/// a fourth shape the compiler knows by name. The same rule reads `abs(x)` over an instant +/// selection, which is why the two are asserted together: nothing below the map is specific to it. +#[test] +fn a_math_function_maps_over_the_node_its_argument_compiles_to() { + let tree = compile( + &eval_stmt("abs(rate(x[5m]))"), + raw_schema(), + SeriesOrdering::KeyThenTime, + ) + .unwrap_or_else(|error| panic!("`abs(rate(x[5m]))` was refused: {error}")); + + assert!( + matches!(map_op(tree.root()), MathOp::Abs), + "the root is not an abs: {:?}", + map_parts(tree.root()).1.ops + ); + let (window, _) = map_parts(tree.root()); + let NativeExpr::Window { input, op } = &window.expr else { + panic!("the map's input is not a window: {:?}", window.expr); + }; + assert_eq!(op.function, RangeFunction::Rate); + assert_eq!(op.window.range_ms, 300_000); + let NativeExpr::Input(bound) = &input.expr else { + panic!("the window's input is not an input: {:?}", input.expr); + }; + assert!( + matches!(&bound.output, NativeType::Samples(schema) if schema.eval_grid.is_none()), + "the window's input is not raw samples: {:?}", + bound.output + ); + assert_eq!(bound.ordering, SeriesOrdering::KeyThenTime); + + // The same rule over the other selection shape: a map over an instant selection. + let tree = compile( + &eval_stmt("abs(x)"), + raw_schema(), + SeriesOrdering::KeyThenTime, + ) + .unwrap_or_else(|error| panic!("`abs(x)` was refused: {error}")); + assert!(matches!(map_op(tree.root()), MathOp::Abs)); + let (instant, _) = map_parts(tree.root()); + assert!( + matches!(instant.expr, NativeExpr::Instant { .. }), + "`abs(x)`'s map is not over an instant selection: {:?}", + instant.expr + ); +} + +/// Nested math functions are two maps over one window, and every map reports that window's layout. +/// +/// The maps are separate nodes because the arguments nest, not because the compiler knows +/// `clamp_min(abs(...))`: a compiler that bound the pair to one node would fail the tree assertions +/// here. The grids are compared by their spec, which is what [`NativeTree::validate`] compares: a +/// map that allocated a second, equal grid would still be the window's layout, while one that +/// reported a different grid or different fields would not. +#[test] +fn nested_math_functions_are_two_maps_over_one_window() { + let tree = compile( + &eval_stmt("clamp_min(abs(rate(x[5m])), 0)"), + raw_schema(), + SeriesOrdering::KeyThenTime, + ) + .unwrap_or_else(|error| panic!("`clamp_min(abs(rate(x[5m])), 0)` was refused: {error}")); + + let MathOp::ClampMin { min } = map_op(tree.root()) else { + panic!( + "the root is not a clamp_min: {:?}", + map_parts(tree.root()).1.ops + ); + }; + assert_eq!( + min, 0.0, + "the clamp bound is not the number the query wrote" + ); + + let (inner, _) = map_parts(tree.root()); + assert!( + matches!(map_op(inner), MathOp::Abs), + "the inner node is not an abs: {:?}", + map_parts(inner).1.ops + ); + let (window, op) = map_parts(inner); + let NativeExpr::Window { op: bound, .. } = &window.expr else { + panic!("the inner map's input is not a window: {:?}", window.expr); + }; + assert_eq!(bound.function, RangeFunction::Rate); + assert_eq!(op.ops.len(), 1, "a map applies one operation"); + + let window_schema = evaluated(window); + let window_grid = window_schema + .eval_grid + .as_ref() + .expect("the window is evaluated"); + for (what, node) in [("the clamp_min", tree.root()), ("the abs", inner)] { + let schema = evaluated(node); + assert_eq!( + schema + .eval_grid + .as_ref() + .expect("the map is evaluated") + .spec(), + window_grid.spec(), + "`{what}` reports a grid the window does not" + ); + assert!( + schema + .fields + .iter() + .map(|field| &field.name) + .eq(window_schema.fields.iter().map(|field| &field.name)), + "`{what}` reports fields the window does not" + ); + } +} + +/// Every listed math name binds its own kernel, with the constants the call wrote. The display names +/// the operation and its constants, so a name that fell through to a neighbour's variant — or a +/// parameter that was bound from the wrong argument — shows up in a line someone would read. +#[test] +fn the_supported_math_functions_map_to_their_kernels() { + let cases = [ + ("abs(x)", "abs"), + ("ceil(x)", "ceil"), + ("floor(x)", "floor"), + ("sqrt(x)", "sqrt"), + ("exp(x)", "exp"), + ("ln(x)", "ln"), + ("log2(x)", "log2"), + ("log10(x)", "log10"), + ("sin(x)", "sin"), + ("cos(x)", "cos"), + ("tan(x)", "tan"), + ("asin(x)", "asin"), + ("acos(x)", "acos"), + ("atan(x)", "atan"), + ("sinh(x)", "sinh"), + ("cosh(x)", "cosh"), + ("tanh(x)", "tanh"), + ("asinh(x)", "asinh"), + ("acosh(x)", "acosh"), + ("atanh(x)", "atanh"), + ("deg(x)", "deg"), + ("rad(x)", "rad"), + ("sgn(x)", "sgn"), + // The three shapes that carry constants, including the default `round` binds one for. + ("round(x)", "round 1"), + ("round(x, 0.5)", "round 0.5"), + ("clamp(x, 0, 1)", "clamp 0 1"), + ("clamp_min(x, 0)", "clamp_min 0"), + ("clamp_max(x, 1)", "clamp_max 1"), + ]; + + for (query, op) in cases { + let tree = compile(&eval_stmt(query), raw_schema(), SeriesOrdering::KeyThenTime) + .unwrap_or_else(|error| panic!("`{query}` was refused: {error}")); + let display = tree.to_string(); + let root = display.lines().nth(1).expect("a compiled tree has a root"); + assert_eq!( + root, + format!(" root Map {op}"), + "`{query}` bound the wrong kernel" + ); + } +} + +/// The constants a function folds are folded by the kernel's own arithmetic, not by a second copy of +/// it: `%` keeps the sign of its left operand and `^` is `go_math`'s power. Both numbers below are +/// ones a hand-written copy would get wrong: a Euclidean modulo folds `-7 % 3` to 2, `*` folds the +/// power to about 9.8e8, and Rust's own `f64::powf` folds it to 3.623449445683154e291 where the +/// kernel gives 3.6234491766672406e291, because `go_math::pow` accumulates error by repeated +/// squaring as Go's `math.Pow` does. That is the point: the compiler must not disagree with the +/// kernel about what an operator means. +#[test] +fn the_constant_fold_is_the_kernels_arithmetic() { + // Go's `math.Mod` gives -1 for `-7 % 3`; a Euclidean modulo would give 2. + let tree = compile( + &eval_stmt("clamp_min(x, (-7) % 3)"), + raw_schema(), + SeriesOrdering::KeyThenTime, + ) + .unwrap_or_else(|error| panic!("`clamp_min(x, (-7) % 3)` was refused: {error}")); + let MathOp::ClampMin { min } = map_op(tree.root()) else { + panic!( + "the root is not a clamp_min: {:?}", + map_parts(tree.root()).1.ops + ); + }; + assert_eq!(min, -1.0, "`-7 % 3` did not fold with `math.Mod`'s sign"); + + // `1.0000006850274261 ^ 9.80019037e8` is a power over a base barely above one: as multiplication + // it would be about 9.8e8, and it is neither a comparison nor a set operator, which fold nothing + // at all. It is also the case `go_math` keeps its own `pow` for: Go's repeated squaring + // accumulates error the standard library's `pow` does not, so the two disagree well above the + // last bit and a fold that hand-wrote `powf` binds a different number here. + let tree = compile( + &eval_stmt("round(x, 1.0000006850274261 ^ 9.80019037e8)"), + raw_schema(), + SeriesOrdering::KeyThenTime, + ) + .unwrap_or_else(|error| { + panic!("`round(x, 1.0000006850274261 ^ 9.80019037e8)` was refused: {error}") + }); + let MathOp::Round { nearest } = map_op(tree.root()) else { + panic!( + "the root is not a round: {:?}", + map_parts(tree.root()).1.ops + ); + }; + assert_eq!( + nearest, 3.6234491766672406e291, + "`1.0000006850274261 ^ 9.80019037e8` did not fold with `go_math`'s power" + ); +} + +/// A math function is pointwise over a vector, so an argument that is not one is refused and the +/// rejection names the argument rather than the function alone. The parser refuses `abs(1)` too, so +/// the AST is edited the way the arity test edits one: a shape the parser would have caught is still +/// a shape the compiler must not compile into something else. +#[test] +fn a_math_function_over_a_non_vector_argument_is_refused_and_names_it() { + let cases = [ + // A scalar, which a pointwise function has no positions to be pointwise over. + ( + with_call_arguments("abs(x)", |args| { + *args[0] = PromExpr::NumberLiteral(NumberLiteral::new(1.0)); + }), + "a call to `abs` whose first argument is `1`, which is a scalar", + ), + // A range selector, which belongs in a window rather than under a map. + ( + with_call_arguments("abs(x)", |args| { + args[0] = a_matrix_selector(); + }), + "a call to `abs` whose first argument is `x[5m]`, which is a matrix", + ), + ]; + + for (stmt, named) in cases { + let error = compile(&stmt, raw_schema(), SeriesOrdering::KeyThenTime) + .expect_err("a math function over a non-vector compiled"); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "the refusal is not NotImplemented: {error}" + ); + let message = error.to_string(); + assert!( + message.contains(named), + "the refusal did not name the argument it found: {message}" + ); + assert!( + message.contains("where the function takes a vector"), + "the refusal did not say what the argument has to be: {message}" + ); + } +}