mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 08:44:07 +00:00
fix(promql): replace the residual hybrid binaries and drop the 64-level depth cap (#10254)
Remove the last hybrid composition: when a native statement is refused as a whole but each side is independently admissible, the sides are now compiled through the same native frame preparation, compiler and placement path the whole-native route uses and meet at a shallow native Binary, instead of the native operands being composed by the legacy planner's Join. Admission is decided by the composition itself - equal grids, the binder's matching/cardinality/fill support, the output column's kind, and the runtime execution state - rather than by a predicate that copied what the removed hybrid route used to accept. The artificial 64-level native depth cap is gone on both the construction and the decoding side, with no replacement cap, no depth-based splitting and no product stack configuration; the recursive walks that actually need it grow their stack on demand (validate, runtime build, runtime pull). Histogram binary results are preserved and their diagnostics aligned with Prometheus: an unsupported pairing synthesized by a fill now reports the same Info, and an incompatible bucket layout reports only the schema Warning instead of also reporting a reset-hint contradiction. The legacy histogram UDFs follow the same order so the two routes stay in step. Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
Generated
+1
@@ -11413,6 +11413,7 @@ dependencies = [
|
||||
"regex",
|
||||
"serde",
|
||||
"snafu 0.8.6",
|
||||
"stacker",
|
||||
"store-api",
|
||||
"table",
|
||||
"tokio",
|
||||
|
||||
@@ -258,6 +258,7 @@ sqlx = { version = "0.8", default-features = false, features = [
|
||||
"tls-rustls-aws-lc-rs",
|
||||
"rust_decimal",
|
||||
] }
|
||||
stacker = "0.1.25"
|
||||
stringprep = "0.1"
|
||||
strum = { version = "0.27", features = ["derive"] }
|
||||
sysinfo = "0.33"
|
||||
|
||||
@@ -34,6 +34,7 @@ prost.workspace = true
|
||||
regex.workspace = true
|
||||
serde = { workspace = true, features = ["rc"] }
|
||||
snafu.workspace = true
|
||||
stacker.workspace = true
|
||||
store-api.workspace = true
|
||||
table.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
@@ -405,18 +405,27 @@ fn histogram_pair_udf_with_collector(
|
||||
result.push(
|
||||
match (read_histogram(lhs, row)?, read_histogram(rhs, row)?) {
|
||||
(Some(lhs), Some(rhs)) => {
|
||||
record_custom_reconciliation(&collector, name, &lhs, &rhs);
|
||||
record_counter_reset_contradiction(&collector, name, &lhs, &rhs);
|
||||
let result = op(&lhs, &rhs);
|
||||
if result.is_none() {
|
||||
record_warning(
|
||||
&collector,
|
||||
format!(
|
||||
"{name}: dropped native histogram sample with incompatible schemas"
|
||||
),
|
||||
);
|
||||
// The layouts decide whether there is a pair at all: Prometheus reports
|
||||
// a pair whose bucket layouts cannot be combined as that one error,
|
||||
// before it looks at the reset hints, so a pair that fails both carries
|
||||
// only the schema warning. This is the order the native matcher in
|
||||
// `vector_match` reports it in too.
|
||||
match op(&lhs, &rhs) {
|
||||
Some(result) => {
|
||||
record_custom_reconciliation(&collector, name, &lhs, &rhs);
|
||||
record_counter_reset_contradiction(&collector, name, &lhs, &rhs);
|
||||
Some(result)
|
||||
}
|
||||
None => {
|
||||
record_warning(
|
||||
&collector,
|
||||
format!(
|
||||
"{name}: dropped native histogram sample with incompatible schemas"
|
||||
),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
@@ -3612,7 +3621,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subtraction_records_reset_hint_contradictions_for_incompatible_histograms() {
|
||||
fn subtraction_reports_a_reset_hint_contradiction_only_between_compatible_histograms() {
|
||||
// Prometheus reports a pair whose bucket layouts cannot be combined before it looks at the
|
||||
// reset hints, so the collision warning belongs to the pairs the operation can answer.
|
||||
for incompatible in [false, true] {
|
||||
let mut left = sample_histogram(2.0, 2.0, vec![2.0]);
|
||||
left.reset_hint = COUNTER_RESET_HINT;
|
||||
@@ -3633,10 +3644,23 @@ mod tests {
|
||||
native_histogram_arrow_type(),
|
||||
);
|
||||
|
||||
assert!(collected_warnings(&collector).contains(&format!(
|
||||
"{}: native histogram counter reset hints contradict",
|
||||
NativeHistogramSub::name()
|
||||
)));
|
||||
let warnings = collected_warnings(&collector);
|
||||
let name = NativeHistogramSub::name();
|
||||
assert_eq!(
|
||||
warnings.contains(&format!(
|
||||
"{name}: native histogram counter reset hints contradict"
|
||||
)),
|
||||
!incompatible,
|
||||
"{incompatible}: {warnings:?}"
|
||||
);
|
||||
if incompatible {
|
||||
assert!(
|
||||
warnings.contains(&format!(
|
||||
"{name}: dropped native histogram sample with incompatible schemas"
|
||||
)),
|
||||
"{warnings:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1458,6 +1458,8 @@ struct BinaryRuntime {
|
||||
node_index: usize,
|
||||
/// Evidence shared with the other output partitions of this fused-series execution.
|
||||
evidence: Option<Arc<DuplicateEvidence>>,
|
||||
/// The query's collector for dropped histogram sample pairs, when requested.
|
||||
annotations: Option<PromqlAnnotationCollector>,
|
||||
/// The kernel and the modifiers the compiler bound it with.
|
||||
kernel: PairKernel,
|
||||
/// The layout each operand is bound to, what a chunk of it is checked against: `0` is the left
|
||||
@@ -1539,6 +1541,7 @@ impl BinaryRuntime {
|
||||
Self {
|
||||
node_index,
|
||||
evidence,
|
||||
annotations: None,
|
||||
kernel,
|
||||
schemas,
|
||||
operands: [RetainedInput::default(), RetainedInput::default()],
|
||||
@@ -1699,6 +1702,7 @@ impl BinaryRuntime {
|
||||
node: self.slot(0).0,
|
||||
evidence,
|
||||
}),
|
||||
self.annotations.as_ref(),
|
||||
&mut reserve,
|
||||
)
|
||||
}
|
||||
@@ -2983,7 +2987,25 @@ impl NodeState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stack one [`RuntimeNode::build`] or [`RuntimeNode::pull`] level needs below it before the
|
||||
/// recursion grows its stack.
|
||||
///
|
||||
/// Both walks recurse once per tree level, and a debug build's frame for one level of either is
|
||||
/// large: measured on a 100-term chain, a `build` level used about 140 KiB and a `pull` level about
|
||||
/// 80 KiB, which is what aborted the shared 8 MiB runtime worker on a tree the compiler admits.
|
||||
/// There is no depth limit to keep that from happening, so the recursion grows its stack instead;
|
||||
/// the reserve is several levels' worth rather than a bare guard page, because it has to hold the
|
||||
/// level that is about to be entered, not only the one that asked.
|
||||
const NATIVE_STACK_RED_ZONE: usize = 512 * 1024;
|
||||
|
||||
/// Stack a growth adds when the reserve above is spent. It holds several levels of the same walk.
|
||||
const NATIVE_STACK_GROWTH: usize = 4 * 1024 * 1024;
|
||||
|
||||
impl RuntimeNode {
|
||||
/// Builds the runtime node for `node` and, recursively, for its inputs.
|
||||
///
|
||||
/// A deep tree is executed rather than refused: there is no depth limit, so what has to fit is
|
||||
/// the recursion, and it grows its stack on demand instead of relying on the caller's.
|
||||
fn build(
|
||||
node: &NativeNode,
|
||||
pool: &Arc<dyn MemoryPool>,
|
||||
@@ -2991,6 +3013,26 @@ impl RuntimeNode {
|
||||
binary_before: &mut usize,
|
||||
location: String,
|
||||
duplicate_evidence: Option<Arc<DuplicateEvidence>>,
|
||||
) -> Result<(Self, bool)> {
|
||||
stacker::maybe_grow(NATIVE_STACK_RED_ZONE, NATIVE_STACK_GROWTH, || {
|
||||
Self::build_within_stack(
|
||||
node,
|
||||
pool,
|
||||
inputs,
|
||||
binary_before,
|
||||
location,
|
||||
duplicate_evidence,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn build_within_stack(
|
||||
node: &NativeNode,
|
||||
pool: &Arc<dyn MemoryPool>,
|
||||
inputs: &mut Vec<InputSlot>,
|
||||
binary_before: &mut usize,
|
||||
location: String,
|
||||
duplicate_evidence: Option<Arc<DuplicateEvidence>>,
|
||||
) -> Result<(Self, bool)> {
|
||||
let metrics = Arc::new(NodeMetrics::default());
|
||||
let (op, shared) = match &node.expr {
|
||||
@@ -3867,7 +3909,13 @@ impl RuntimeNode {
|
||||
RuntimeOp::VectorOf { input, .. }
|
||||
| RuntimeOp::Instant { input, .. }
|
||||
| RuntimeOp::Merge { input, .. } => input.set_annotations(collector),
|
||||
RuntimeOp::Binary { left, right, .. } => {
|
||||
RuntimeOp::Binary {
|
||||
left,
|
||||
right,
|
||||
binary,
|
||||
..
|
||||
} => {
|
||||
binary.annotations = Some(collector.clone());
|
||||
left.set_annotations(collector.clone());
|
||||
right.set_annotations(collector);
|
||||
}
|
||||
@@ -3926,10 +3974,24 @@ impl RuntimeNode {
|
||||
/// `inputs_finished` is the caller's word that no more source chunks will arrive. A node that
|
||||
/// decides nothing until its whole input is there reads it to tell the last pull that found
|
||||
/// nothing -- "nothing more" -- from the ones before it, which meant "nothing yet".
|
||||
///
|
||||
/// The pull recurses once per tree level, so a deep tree grows the stack it runs on rather than
|
||||
/// being refused ([`NATIVE_STACK_RED_ZONE`]). The stack is the driver's, whichever thread polls
|
||||
/// the result stream: a shared runtime worker has 8 MiB of it in a debug build.
|
||||
fn pull(
|
||||
&mut self,
|
||||
inputs: &mut [InputSlot],
|
||||
inputs_finished: bool,
|
||||
) -> Result<Option<NativeChunk>> {
|
||||
stacker::maybe_grow(NATIVE_STACK_RED_ZONE, NATIVE_STACK_GROWTH, || {
|
||||
self.pull_within_stack(inputs, inputs_finished)
|
||||
})
|
||||
}
|
||||
|
||||
fn pull_within_stack(
|
||||
&mut self,
|
||||
inputs: &mut [InputSlot],
|
||||
inputs_finished: bool,
|
||||
) -> Result<Option<NativeChunk>> {
|
||||
match &mut self.op {
|
||||
// A source hands over what the caller queued at it. The id was assigned when the tree
|
||||
|
||||
@@ -934,6 +934,98 @@ fn run(executor: &mut NativeExecutor) -> Vec<EvaluatedChunk> {
|
||||
out
|
||||
}
|
||||
|
||||
/// A deep native tree executes after the real FusedSeries envelope is decoded.
|
||||
#[test]
|
||||
fn deep_native_tree_executes_after_fused_series_round_trip() {
|
||||
// A representative deep codec/runtime test owns its worker stack; this is not an
|
||||
// arbitrary-depth guarantee or a product-level stack setting.
|
||||
std::thread::Builder::new()
|
||||
.name("deep-native-codec".into())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
use crate::extension_plan::{FusedOutputFormat, FusedSeries, FusedSeriesBinding};
|
||||
use datafusion::common::DFSchema;
|
||||
use datafusion::logical_expr::{EmptyRelation, LogicalPlan};
|
||||
|
||||
let grid = grid(0, 60_000, 3);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let mut root = evaluated_input(Arc::clone(&grid));
|
||||
for _ in 0..80 {
|
||||
root = scalar_binary(
|
||||
root,
|
||||
scalar(1.0, Arc::clone(&grid)),
|
||||
binary_matching(BinaryOp::Add),
|
||||
);
|
||||
}
|
||||
root = map(
|
||||
root,
|
||||
vec![MathOp::Clamp {
|
||||
min: 0.0,
|
||||
max: 1000.0,
|
||||
}],
|
||||
);
|
||||
let tree = NativeTree::try_new(root).unwrap();
|
||||
let output_schema = FusedSeries::series_frame_schema(&tree).unwrap();
|
||||
let output_schema =
|
||||
Arc::new(DFSchema::try_from(output_schema.as_ref().clone()).unwrap());
|
||||
let node = FusedSeries::try_new_with_format(
|
||||
Arc::from("deep"),
|
||||
tree,
|
||||
FusedSeriesBinding {
|
||||
grid: Arc::clone(&grid),
|
||||
time_index: "ts".to_string(),
|
||||
tags: Vec::new(),
|
||||
metric_name: None,
|
||||
},
|
||||
vec![LogicalPlan::EmptyRelation(EmptyRelation {
|
||||
produce_one_row: false,
|
||||
schema: Arc::new(DFSchema::empty()),
|
||||
})],
|
||||
output_schema,
|
||||
FusedOutputFormat::SeriesFrame,
|
||||
)
|
||||
.unwrap();
|
||||
let bytes = node.serialize().unwrap();
|
||||
let decoded = FusedSeries::deserialize(&bytes).unwrap();
|
||||
fn depth(node: &NativeNode) -> usize {
|
||||
1 + match &node.expr {
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) => 0,
|
||||
NativeExpr::Map { input, .. } => depth(input),
|
||||
NativeExpr::Binary { left, right, .. } => depth(left).max(depth(right)),
|
||||
other => panic!("unexpected node: {other:?}"),
|
||||
}
|
||||
}
|
||||
assert!(depth(decoded.tree().root()) > 64);
|
||||
let pool = pool();
|
||||
let mut execution = executor(decoded.tree(), Arc::clone(&pool));
|
||||
execution
|
||||
.push_evaluated(
|
||||
InputId::new(0),
|
||||
evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("x", "a"),
|
||||
0,
|
||||
vec![Some(-1.0), None, Some(3.0)],
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let chunks = run(&mut execution);
|
||||
assert_eq!(chunks.len(), 1);
|
||||
assert_eq!(
|
||||
sample_values(&chunks[0]),
|
||||
vec![Some(79.0), None, Some(83.0)]
|
||||
);
|
||||
assert_eq!(chunks[0].series.time.grid().unwrap().spec(), grid.spec());
|
||||
assert_eq!(chunks[0].series.metric.get("job"), Some("a"));
|
||||
assert_eq!(chunks[0].series.metric.name(), None);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// A scalar root answers its own value, once, on the grid it was bound to.
|
||||
///
|
||||
/// It reads nothing, so what it reports is not a series it was handed: one unlabelled chunk holding
|
||||
@@ -7208,6 +7300,633 @@ fn a_binary_adds_the_native_histograms_of_both_sides() {
|
||||
assert_eq!(histogram_counts(&out[0]), vec![8.0, 8.0]);
|
||||
}
|
||||
|
||||
/// The native vector matcher annotates only unsupported, present histogram pairs.
|
||||
#[test]
|
||||
fn binary_histogram_drop_info_respects_presence_and_duplicates() {
|
||||
use common_query::promql_annotations::PromqlAnnotationCollector;
|
||||
|
||||
let infos = |collector: &PromqlAnnotationCollector| {
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
assert!(warnings.is_empty());
|
||||
infos
|
||||
};
|
||||
for (op, histogram_left, histogram_right, symbol) in [
|
||||
(BinaryOp::Add, true, false, "+"),
|
||||
(BinaryOp::Add, false, true, "+"),
|
||||
(BinaryOp::Mul, true, true, "*"),
|
||||
] {
|
||||
let (tree, grid, left, right) = binary_tree(op, 2);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state = NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
let chunk = |histogram, name| {
|
||||
if histogram {
|
||||
evaluated_histograms(&schema, named_metric(name, "a"), 0, 4, 2, true)
|
||||
} else {
|
||||
evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric(name, "a"),
|
||||
0,
|
||||
vec![Some(2.0), Some(2.0)],
|
||||
true,
|
||||
)
|
||||
}
|
||||
};
|
||||
execution
|
||||
.push_evaluated(left, chunk(histogram_left, "left"))
|
||||
.unwrap();
|
||||
execution
|
||||
.push_evaluated(right, chunk(histogram_right, "right"))
|
||||
.unwrap();
|
||||
assert!(run(&mut execution).is_empty());
|
||||
assert_eq!(
|
||||
infos(&collector),
|
||||
vec![format!(
|
||||
"{symbol}: dropped native histogram samples because this binary operation is not supported for native histograms"
|
||||
)]
|
||||
);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
// No label match, or a missing sample at every shared point, cannot produce a drop Info.
|
||||
for unmatched in [false, true] {
|
||||
let (tree, grid, left, right) = binary_tree(BinaryOp::Add, 2);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let mut state = NativeExecutionState::try_new(&tree, pool(), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
execution
|
||||
.push_evaluated(
|
||||
left,
|
||||
evaluated_histograms(&schema, named_metric("hist", "a"), 0, 4, 2, true),
|
||||
)
|
||||
.unwrap();
|
||||
let values = if unmatched {
|
||||
vec![Some(2.0); 2]
|
||||
} else {
|
||||
vec![None; 2]
|
||||
};
|
||||
let job = if unmatched { "b" } else { "a" };
|
||||
execution
|
||||
.push_evaluated(
|
||||
right,
|
||||
evaluated_chunk_of(&schema, named_metric("float", job), 0, values, true),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(run(&mut execution).is_empty());
|
||||
assert!(infos(&collector).is_empty());
|
||||
}
|
||||
|
||||
// Even when the operation would drop the pair, duplicate one-side series retain their error.
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let mut matching = binary_matching(BinaryOp::Add);
|
||||
matching.labels = MatchingLabels::On(vec![Arc::from("job")]);
|
||||
let tree = NativeTree::try_new(binary_of(
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 0),
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 1),
|
||||
matching,
|
||||
))
|
||||
.unwrap();
|
||||
let left = InputId::new(0);
|
||||
let right = InputId::new(1);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state = NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
execution
|
||||
.push_evaluated(
|
||||
left,
|
||||
evaluated_histograms(&schema, pod_metric("hist", "a", "h"), 0, 4, 1, true),
|
||||
)
|
||||
.unwrap();
|
||||
for pod in ["one", "two"] {
|
||||
execution
|
||||
.push_evaluated(
|
||||
right,
|
||||
evaluated_chunk_of(
|
||||
&schema,
|
||||
pod_metric("float", "a", pod),
|
||||
0,
|
||||
vec![Some(2.0)],
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let error = execution.finish().unwrap_err().to_string();
|
||||
assert!(
|
||||
error.contains("duplicate series for the match group"),
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// A filled pair is a pair the operation is run on, so the native executor annotates the
|
||||
/// unsupported histogram pairing a fill value made.
|
||||
///
|
||||
/// `hist{job="a"}` has no `job="b"` counterpart, so `hist + on(job) fill_right(0) float` answers
|
||||
/// every point of it with `histogram + 0`, the pairing Prometheus runs `doBinOp` on after it fills
|
||||
/// a step. The Info it records is the wording the legacy planner states the same rule with, and a
|
||||
/// pair the fill never forms at all carries nothing.
|
||||
#[test]
|
||||
fn binary_histogram_fill_reports_the_dropped_info() {
|
||||
use common_query::promql_annotations::PromqlAnnotationCollector;
|
||||
|
||||
let dropped = |symbol: &str| {
|
||||
vec![format!(
|
||||
"{symbol}: dropped native histogram samples because this binary operation is not supported for native histograms"
|
||||
)]
|
||||
};
|
||||
// `fill_right` answers a missing right-hand sample, so the histogram is the left operand and
|
||||
// every point of it is the pair `histogram + 0`; `fill_left` answers a missing left-hand one,
|
||||
// so the histogram is the right operand and the pair is `0 + histogram`.
|
||||
for (fill, histogram_left) in [
|
||||
(
|
||||
FillValues {
|
||||
lhs: None,
|
||||
rhs: Some(0.0),
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
FillValues {
|
||||
lhs: Some(0.0),
|
||||
rhs: None,
|
||||
},
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let grid = grid(0, 60_000, 2);
|
||||
let mut matching = binary_matching(BinaryOp::Add);
|
||||
matching.labels = MatchingLabels::On(vec![Arc::from("job")]);
|
||||
matching.fill = fill;
|
||||
let tree = NativeTree::try_new(binary_of(
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 0),
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 1),
|
||||
matching,
|
||||
))
|
||||
.unwrap();
|
||||
let schema = evaluated_schema(grid);
|
||||
// The two sides carry different `job` labels, so nothing matches and the fill value is what
|
||||
// every point of the histogram series pairs with.
|
||||
let histogram = || evaluated_histograms(&schema, named_metric("hist", "a"), 0, 4, 2, true);
|
||||
let float = || {
|
||||
evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("float", "b"),
|
||||
0,
|
||||
vec![Some(2.0), Some(2.0)],
|
||||
true,
|
||||
)
|
||||
};
|
||||
let (left, right) = if histogram_left {
|
||||
(histogram(), float())
|
||||
} else {
|
||||
(float(), histogram())
|
||||
};
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state = NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
execution.push_evaluated(InputId::new(0), left).unwrap();
|
||||
execution.push_evaluated(InputId::new(1), right).unwrap();
|
||||
let chunks = run(&mut execution);
|
||||
assert!(
|
||||
chunks.is_empty(),
|
||||
"a dropped fill pair filed a row: {:?}",
|
||||
chunks
|
||||
.iter()
|
||||
.map(|chunk| chunk.series.metric.clone())
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
assert!(warnings.is_empty(), "{warnings:?}");
|
||||
assert_eq!(infos, dropped("+"), "{fill:?}");
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
// Without a fill modifier the two sides never pair, and a pair that was never formed produces
|
||||
// no sample to drop.
|
||||
let grid = grid(0, 60_000, 2);
|
||||
let mut matching = binary_matching(BinaryOp::Add);
|
||||
matching.labels = MatchingLabels::On(vec![Arc::from("job")]);
|
||||
let tree = NativeTree::try_new(binary_of(
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 0),
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 1),
|
||||
matching,
|
||||
))
|
||||
.unwrap();
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state = NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
execution
|
||||
.push_evaluated(
|
||||
InputId::new(0),
|
||||
evaluated_histograms(&schema, named_metric("hist", "a"), 0, 4, 2, true),
|
||||
)
|
||||
.unwrap();
|
||||
execution
|
||||
.push_evaluated(
|
||||
InputId::new(1),
|
||||
evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("float", "b"),
|
||||
0,
|
||||
vec![Some(2.0), Some(2.0)],
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(run(&mut execution).is_empty());
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
assert!(warnings.is_empty(), "{warnings:?}");
|
||||
assert!(infos.is_empty(), "{infos:?}");
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// Matched histogram arithmetic preserves the existing reset/schema diagnostics and values.
|
||||
#[test]
|
||||
fn binary_histogram_add_sub_preserves_payload_and_diagnostics() {
|
||||
use common_query::native_histogram::CUSTOM_BUCKETS_SCHEMA;
|
||||
use common_query::promql_annotations::PromqlAnnotationCollector;
|
||||
|
||||
// A single positive bucket makes its post-reconciliation count unambiguous, including when
|
||||
// all custom bounds differ and both source buckets fold into the result's +Inf bucket.
|
||||
let hist = |custom: bool, bounds: &[f64], hint, count, sum| {
|
||||
let offsets = [0];
|
||||
let lengths = [1];
|
||||
let buckets = [count];
|
||||
let mut builder = HistogramBatchBuilder::new();
|
||||
let index = builder
|
||||
.push(HistogramView {
|
||||
schema: if custom { CUSTOM_BUCKETS_SCHEMA } else { 0 },
|
||||
zero_threshold: 0.0,
|
||||
sum,
|
||||
count,
|
||||
zero_count: 0.0,
|
||||
reset_hint: hint,
|
||||
start_timestamp: None,
|
||||
custom_values: bounds,
|
||||
positive: BucketSideView {
|
||||
span_offsets: &offsets,
|
||||
span_lengths: &lengths,
|
||||
buckets: &buckets,
|
||||
},
|
||||
negative: BucketSideView {
|
||||
span_offsets: &[],
|
||||
span_lengths: &[],
|
||||
buckets: &[],
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
(builder.finish().unwrap(), index)
|
||||
};
|
||||
for op in [BinaryOp::Add, BinaryOp::Sub] {
|
||||
let name = if op == BinaryOp::Add {
|
||||
"prom_native_histogram_add"
|
||||
} else {
|
||||
"prom_native_histogram_sub"
|
||||
};
|
||||
let factor = if op == BinaryOp::Add { 1.0 } else { -1.0 };
|
||||
for reverse in [false, true] {
|
||||
// Same exponential buckets; both orientations of the reset contradiction are tested.
|
||||
let hints = if reverse {
|
||||
(
|
||||
CounterResetHint::NotCounterReset,
|
||||
CounterResetHint::CounterReset,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
CounterResetHint::CounterReset,
|
||||
CounterResetHint::NotCounterReset,
|
||||
)
|
||||
};
|
||||
let left_hist = hist(false, &[], hints.0, 5.0, 10.0);
|
||||
let right_hist = hist(false, &[], hints.1, 2.0, 4.0);
|
||||
let (tree, grid, left, right) = binary_tree(op, 1);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state = NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for (input, (table, index)) in [(left, left_hist), (right, right_hist)] {
|
||||
execution
|
||||
.push_evaluated(
|
||||
input,
|
||||
evaluated_histogram_chunk_of(
|
||||
&schema,
|
||||
named_metric("hist", "a"),
|
||||
0,
|
||||
vec![Some(index)],
|
||||
&table,
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let chunks = run(&mut execution);
|
||||
assert_eq!(chunks.len(), 1);
|
||||
let mut cursor = chunks[0].series.fields.as_slice()[0].cursor();
|
||||
let SampleRef::Histogram(h) = cursor.at(0).unwrap() else {
|
||||
panic!("expected histogram")
|
||||
};
|
||||
assert_eq!(
|
||||
(h.count, h.sum, h.zero_count),
|
||||
(5.0 + factor * 2.0, 10.0 + factor * 4.0, 0.0)
|
||||
);
|
||||
assert_eq!(h.positive.buckets, &[5.0 + factor * 2.0]);
|
||||
assert_eq!(
|
||||
h.reset_hint,
|
||||
if op == BinaryOp::Add {
|
||||
CounterResetHint::Unknown
|
||||
} else {
|
||||
CounterResetHint::Gauge
|
||||
}
|
||||
);
|
||||
assert_eq!(chunks[0].series.metric.get("job"), Some("a"));
|
||||
assert_eq!(chunks[0].series.metric.name(), None);
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
assert_eq!(
|
||||
warnings,
|
||||
vec![format!(
|
||||
"{name}: native histogram counter reset hints contradict"
|
||||
)]
|
||||
);
|
||||
assert!(infos.is_empty());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
// Differing custom bounds reconcile; exercise partially overlapping and disjoint bounds,
|
||||
// with and without contradictory reset hints, so each existing diagnostic is independent.
|
||||
for (left_bounds, right_bounds) in
|
||||
[(&[1.0, 2.0][..], &[2.0, 3.0][..]), (&[1.0][..], &[2.0][..])]
|
||||
{
|
||||
for collision in [false, true] {
|
||||
let hints = if collision {
|
||||
(
|
||||
CounterResetHint::CounterReset,
|
||||
CounterResetHint::NotCounterReset,
|
||||
)
|
||||
} else {
|
||||
(CounterResetHint::Unknown, CounterResetHint::Unknown)
|
||||
};
|
||||
let left_hist = hist(true, left_bounds, hints.0, 5.0, 10.0);
|
||||
let right_hist = hist(true, right_bounds, hints.1, 2.0, 4.0);
|
||||
let (tree, grid, left, right) = binary_tree(op, 1);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state =
|
||||
NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for (input, (table, index)) in [(left, left_hist), (right, right_hist)] {
|
||||
execution
|
||||
.push_evaluated(
|
||||
input,
|
||||
evaluated_histogram_chunk_of(
|
||||
&schema,
|
||||
named_metric("hist", "a"),
|
||||
0,
|
||||
vec![Some(index)],
|
||||
&table,
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let chunks = run(&mut execution);
|
||||
assert_eq!(chunks.len(), 1);
|
||||
let mut cursor = chunks[0].series.fields.as_slice()[0].cursor();
|
||||
let SampleRef::Histogram(h) = cursor.at(0).unwrap() else {
|
||||
panic!("expected histogram")
|
||||
};
|
||||
assert_eq!((h.count, h.sum), (5.0 + factor * 2.0, 10.0 + factor * 4.0));
|
||||
assert_eq!(h.positive.buckets, &[5.0 + factor * 2.0]);
|
||||
assert_eq!(
|
||||
h.custom_values,
|
||||
if left_bounds.len() == 1 {
|
||||
&[][..]
|
||||
} else {
|
||||
&[2.0][..]
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
h.reset_hint,
|
||||
if op == BinaryOp::Add {
|
||||
CounterResetHint::Unknown
|
||||
} else {
|
||||
CounterResetHint::Gauge
|
||||
}
|
||||
);
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
let expected_warnings = if collision {
|
||||
vec![format!(
|
||||
"{name}: native histogram counter reset hints contradict"
|
||||
)]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
assert_eq!(warnings, expected_warnings);
|
||||
assert_eq!(
|
||||
infos,
|
||||
vec![format!(
|
||||
"{name}: reconciled native histograms with different custom buckets"
|
||||
)]
|
||||
);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
// A conflicting pair is not a sample pair at an absent point or when labels do not match.
|
||||
for unmatched in [false, true] {
|
||||
let left_hist = hist(true, &[1.0], CounterResetHint::CounterReset, 5.0, 10.0);
|
||||
let right_hist = hist(false, &[], CounterResetHint::NotCounterReset, 2.0, 4.0);
|
||||
let (tree, grid, left, right) = binary_tree(op, 1);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state = NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
execution
|
||||
.push_evaluated(
|
||||
left,
|
||||
evaluated_histogram_chunk_of(
|
||||
&schema,
|
||||
named_metric("left", "a"),
|
||||
0,
|
||||
vec![Some(left_hist.1)],
|
||||
&left_hist.0,
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let right_index = if unmatched { Some(right_hist.1) } else { None };
|
||||
let right_job = if unmatched { "b" } else { "a" };
|
||||
execution
|
||||
.push_evaluated(
|
||||
right,
|
||||
evaluated_histogram_chunk_of(
|
||||
&schema,
|
||||
named_metric("right", right_job),
|
||||
0,
|
||||
vec![right_index],
|
||||
&right_hist.0,
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(run(&mut execution).is_empty());
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
assert!(warnings.is_empty());
|
||||
assert!(infos.is_empty());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
// Incompatible custom/exponential pairs are dropped, in either direction. Prometheus
|
||||
// reports the layouts before it reaches the reset hints, so a pair that fails both carries
|
||||
// the schema warning alone.
|
||||
for reverse in [false, true] {
|
||||
for collision in [false, true] {
|
||||
let hints = if collision {
|
||||
(
|
||||
CounterResetHint::CounterReset,
|
||||
CounterResetHint::NotCounterReset,
|
||||
)
|
||||
} else {
|
||||
(CounterResetHint::Unknown, CounterResetHint::Unknown)
|
||||
};
|
||||
let left_hist = hist(
|
||||
reverse,
|
||||
if reverse { &[1.0] } else { &[] },
|
||||
hints.0,
|
||||
5.0,
|
||||
10.0,
|
||||
);
|
||||
let right_hist = hist(
|
||||
!reverse,
|
||||
if reverse { &[] } else { &[1.0] },
|
||||
hints.1,
|
||||
2.0,
|
||||
4.0,
|
||||
);
|
||||
let (tree, grid, left, right) = binary_tree(op, 1);
|
||||
let schema = evaluated_schema(grid);
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let pool = pool();
|
||||
let mut state =
|
||||
NativeExecutionState::try_new(&tree, Arc::clone(&pool), None).unwrap();
|
||||
state.set_annotations(Some(collector.clone()));
|
||||
let mut execution = NativeExecutor::new(state);
|
||||
for (input, (table, index)) in [(left, left_hist), (right, right_hist)] {
|
||||
execution
|
||||
.push_evaluated(
|
||||
input,
|
||||
evaluated_histogram_chunk_of(
|
||||
&schema,
|
||||
named_metric("hist", "a"),
|
||||
0,
|
||||
vec![Some(index)],
|
||||
&table,
|
||||
true,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
assert!(run(&mut execution).is_empty());
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
// The hint collision changes nothing here: the layouts already decided the pair,
|
||||
// and that one warning is all Prometheus reports for it.
|
||||
assert_eq!(
|
||||
warnings,
|
||||
vec![format!(
|
||||
"{name}: dropped native histogram sample with incompatible schemas"
|
||||
)],
|
||||
"collision {collision}"
|
||||
);
|
||||
assert!(infos.is_empty());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Supported histogram scaling keeps the values, labels and grid in both operand orders.
|
||||
#[test]
|
||||
fn binary_histogram_scaling_preserves_payload_and_labels() {
|
||||
for (op, histogram_left, expected) in [
|
||||
(BinaryOp::Mul, true, 8.0),
|
||||
(BinaryOp::Mul, false, 8.0),
|
||||
(BinaryOp::Div, true, 2.0),
|
||||
] {
|
||||
let (tree, grid, left, right) = binary_tree(op, 2);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let pool = pool();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let histogram = evaluated_histograms(&schema, named_metric("hist", "a"), 0, 4, 2, true);
|
||||
let float = evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("float", "a"),
|
||||
0,
|
||||
vec![Some(2.0); 2],
|
||||
true,
|
||||
);
|
||||
let (lhs, rhs) = if histogram_left {
|
||||
(histogram, float)
|
||||
} else {
|
||||
(float, histogram)
|
||||
};
|
||||
execution.push_evaluated(left, lhs).unwrap();
|
||||
execution.push_evaluated(right, rhs).unwrap();
|
||||
let chunks = run(&mut execution);
|
||||
assert_eq!(chunks.len(), 1);
|
||||
assert_eq!(histogram_counts(&chunks[0]), vec![expected; 2]);
|
||||
let mut cursor = chunks[0].series.fields.as_slice()[0].cursor();
|
||||
for row in 0..2 {
|
||||
let SampleRef::Histogram(histogram) = cursor.at(row).unwrap() else {
|
||||
panic!("scaled sample is not a histogram");
|
||||
};
|
||||
assert_eq!(histogram.sum, expected);
|
||||
let bucket = if op == BinaryOp::Mul { 2.0 } else { 0.5 };
|
||||
assert_eq!(histogram.positive.buckets, &[bucket; 4]);
|
||||
}
|
||||
assert_eq!(
|
||||
chunks[0].series.time.timestamps().unwrap().as_ref(),
|
||||
&grid.timestamps().unwrap()[..]
|
||||
);
|
||||
assert_eq!(chunks[0].series.metric.get("job"), Some("a"));
|
||||
assert_eq!(chunks[0].series.metric.name(), None);
|
||||
assert_eq!(chunks[0].series.time.grid().unwrap().spec(), grid.spec());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Nothing is decided before the input ends; `finish` is what hands the result over.
|
||||
#[test]
|
||||
fn a_binary_yields_nothing_until_the_input_finishes() {
|
||||
|
||||
+141
-95
@@ -27,36 +27,9 @@
|
||||
//! constant, so `time()` and `scalar(v)` have no node. The legality below is therefore about the
|
||||
//! operations that exist, not about PromQL's whole surface.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::hash::Hash;
|
||||
use std::sync::Arc;
|
||||
|
||||
const MAX_NATIVE_NODE_DEPTH: usize = 64;
|
||||
thread_local! {
|
||||
static NATIVE_NODE_DEPTH: Cell<usize> = const { Cell::new(0) };
|
||||
}
|
||||
|
||||
struct NativeNodeDepthGuard;
|
||||
impl NativeNodeDepthGuard {
|
||||
fn enter<E: serde::de::Error>() -> std::result::Result<Self, E> {
|
||||
NATIVE_NODE_DEPTH.with(|depth| {
|
||||
let current = depth.get();
|
||||
if current >= MAX_NATIVE_NODE_DEPTH {
|
||||
return Err(E::custom(format!(
|
||||
"native tree nesting exceeds {MAX_NATIVE_NODE_DEPTH}"
|
||||
)));
|
||||
}
|
||||
depth.set(current + 1);
|
||||
Ok(Self)
|
||||
})
|
||||
}
|
||||
}
|
||||
impl Drop for NativeNodeDepthGuard {
|
||||
fn drop(&mut self) {
|
||||
NATIVE_NODE_DEPTH.with(|depth| depth.set(depth.get() - 1));
|
||||
}
|
||||
}
|
||||
|
||||
use datafusion::arrow::datatypes::DataType;
|
||||
use datafusion::error::{DataFusionError, Result};
|
||||
use datatypes::series::aggregate_state::{AggregateStateKind, StateComponent};
|
||||
@@ -74,7 +47,9 @@ use crate::map_label::LabelRewrite;
|
||||
use crate::map_time::DateFunction;
|
||||
use crate::range_eval::{RangeFunction, RangeWindow};
|
||||
use crate::select::SortOrder;
|
||||
use crate::vector_match::{BinaryMatching, BinaryOp, Cardinality, MatchingLabels, SetOp};
|
||||
use crate::vector_match::{
|
||||
BinaryMatching, BinaryOp, Cardinality, FillValues, MatchingLabels, SetOp,
|
||||
};
|
||||
|
||||
/// The bucket-bound label a classic histogram's series carry, which the histogram function folds and
|
||||
/// its result does not report.
|
||||
@@ -90,12 +65,14 @@ const PROMQL_METADATA_LABELS: [&str; 3] = ["__name__", "__type__", "__unit__"];
|
||||
/// by `on` or `ignoring`; `and` and `unless` keep the left side's series and `or` adds the right
|
||||
/// side's; `info` adds the data labels of the info table; an operation with a scalar or a function
|
||||
/// of each value keeps its operand's; a scalar and `vector(s)` have none; `count_values` keeps its
|
||||
/// grouping and the label it writes. `None` for a node whose labels this walk does not derive - a
|
||||
/// label rewrite of the metric name, an `absent` naming a label no input column carries, and a
|
||||
/// `count_values` whose destination is the metric name or no label at all, none of which is a column
|
||||
/// here - so a tree holding one reports no label set here. `None` is not the query's answer: the
|
||||
/// caller asks the existing planner for the columns the node is named after instead of declining the
|
||||
/// query.
|
||||
/// grouping and the label it writes; a fill that answers a missing many-side sample files the one
|
||||
/// side's matching labels, so they join the set, while a grouped `include` label joins them only when
|
||||
/// the one side carries it or the matching keeps it in the key a fill reads. `None` for a node whose
|
||||
/// labels this walk does not derive - a label rewrite of the metric name, an `absent` naming a label
|
||||
/// no input column carries, and a `count_values` whose destination is the metric name or no label at
|
||||
/// all, none of which is a column here - so a tree holding one reports no label set here. `None` is
|
||||
/// not the query's answer: the caller asks the existing planner for the columns the node is named
|
||||
/// after instead of declining the query.
|
||||
pub fn fused_output_tags(node: &NativeNode, source_tags: &[Vec<String>]) -> Option<Vec<String>> {
|
||||
match &node.expr {
|
||||
NativeExpr::Input(input) => source_tags.get(input.source_index).cloned(),
|
||||
@@ -286,10 +263,57 @@ pub fn fused_output_tags(node: &NativeNode, source_tags: &[Vec<String>]) -> Opti
|
||||
tags.push(tag.clone());
|
||||
}
|
||||
}
|
||||
// A filled left side takes an unmatched right series' matching labels, so those
|
||||
// results carry the right side's tags.
|
||||
if matching.fill.lhs.is_some() {
|
||||
for tag in kept(fused_output_tags(right, source_tags)?) {
|
||||
// A grouped result carries an include label only when a row can give it one.
|
||||
// `result_metric` removes the label from the many side's labels unless the one side
|
||||
// has it, so a paired row and a row filed from a one-side series take it from the
|
||||
// one series, which is what the loop above declares. A row filed from a many-side
|
||||
// series takes it from that series' match key instead, so it carries the label only
|
||||
// when the matching keeps the label in the key. Everything else the many side
|
||||
// carries is the base above, so a label neither of those can provide has to leave
|
||||
// the set: nothing would ever fill its column.
|
||||
if !includes.is_empty() {
|
||||
let many_rows_filled = match &matching.cardinality {
|
||||
// `group_right` puts the one side on the left, so the fills trade places:
|
||||
// `fill_left` is the one that answers a missing one-side sample there.
|
||||
Cardinality::OneToMany(_) => matching.fill.lhs,
|
||||
Cardinality::ManyToOne(_) | Cardinality::OneToOne => matching.fill.rhs,
|
||||
}
|
||||
.is_some();
|
||||
let in_match_key = |label: &str| match &matching.labels {
|
||||
MatchingLabels::On(names) => {
|
||||
names.iter().any(|name| name.as_ref() == label)
|
||||
}
|
||||
// A match key is built from a series' tags, which never hold the metric name.
|
||||
MatchingLabels::Ignoring(names) => {
|
||||
label != "__name__" && !names.iter().any(|name| name.as_ref() == label)
|
||||
}
|
||||
};
|
||||
tags.retain(|tag| {
|
||||
// The name is not an ordinary label here: this walk reports it by
|
||||
// `__name__` in the set, which the nodes above decide.
|
||||
tag == "__name__"
|
||||
|| !includes.iter().any(|label| label.as_ref() == tag.as_str())
|
||||
|| include_tags.contains(tag)
|
||||
|| (many_rows_filled && in_match_key(tag))
|
||||
});
|
||||
}
|
||||
// A fill answers a missing sample from the series of the other side, and the kernel
|
||||
// files that row at the label set the series it has gives it
|
||||
// (`vector_match::binary_field`): a many-side series keeps all of its labels, which
|
||||
// `tags` already holds, while a one-side series keeps only its matching labels plus
|
||||
// the group labels the modifier names, which the loop above adds. So what a fill can
|
||||
// add here is the one side's matching labels, and only a fill that can file a
|
||||
// one-side row has any to add: one where the many side is the missing side. Which
|
||||
// fill that is follows the kernel, which trades the two for `group_right`, whose one
|
||||
// side is the left - there it is `fill_right` that leaves the many side missing.
|
||||
let (one_tags, fills_many_side) = match &matching.cardinality {
|
||||
Cardinality::OneToMany(_) => (&left_tags, matching.fill.rhs),
|
||||
Cardinality::ManyToOne(_) | Cardinality::OneToOne => {
|
||||
(&right_tags, matching.fill.lhs)
|
||||
}
|
||||
};
|
||||
if fills_many_side.is_some() {
|
||||
for tag in kept(one_tags.clone()) {
|
||||
if !tags.contains(&tag) {
|
||||
tags.push(tag);
|
||||
}
|
||||
@@ -421,6 +445,57 @@ pub fn uniform_field_kind<'a>(
|
||||
answer
|
||||
}
|
||||
|
||||
/// The output kind of a binary operation over two evaluated vectors, including fill pairs.
|
||||
pub fn binary_output_field_kind(
|
||||
left: FieldKind,
|
||||
right: FieldKind,
|
||||
op: BinaryOp,
|
||||
return_bool: bool,
|
||||
fill: FillValues,
|
||||
) -> Option<FieldKind> {
|
||||
if return_bool {
|
||||
return Some(FieldKind::Float);
|
||||
}
|
||||
match op {
|
||||
BinaryOp::Add
|
||||
| BinaryOp::Sub
|
||||
| BinaryOp::Mul
|
||||
| BinaryOp::Div
|
||||
| BinaryOp::Mod
|
||||
| BinaryOp::Pow
|
||||
| BinaryOp::Atan2 => {
|
||||
// An unsupported pair emits no sample, not a float. Fill values are floats, so one
|
||||
// field can be produced by the real pair and by either enabled synthetic pair.
|
||||
let payload_kind = |left, right| match (left, op, right) {
|
||||
(FieldKind::Float, _, FieldKind::Float) => Some(FieldKind::Float),
|
||||
(FieldKind::Histogram, BinaryOp::Add | BinaryOp::Sub, FieldKind::Histogram)
|
||||
| (FieldKind::Histogram, BinaryOp::Mul | BinaryOp::Div, FieldKind::Float)
|
||||
| (FieldKind::Float, BinaryOp::Mul, FieldKind::Histogram) => {
|
||||
Some(FieldKind::Histogram)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let candidates = [
|
||||
payload_kind(left, right),
|
||||
fill.rhs.and_then(|_| payload_kind(left, FieldKind::Float)),
|
||||
fill.lhs.and_then(|_| payload_kind(FieldKind::Float, right)),
|
||||
];
|
||||
let mut kind = None;
|
||||
for candidate in candidates.into_iter().flatten() {
|
||||
if kind.is_some_and(|existing| existing != candidate) {
|
||||
return None;
|
||||
}
|
||||
kind = Some(candidate);
|
||||
}
|
||||
// With no possible samples, retain the legacy float layout for an empty result.
|
||||
Some(kind.unwrap_or(FieldKind::Float))
|
||||
}
|
||||
// Comparison and bool behavior is separate from arithmetic's empty-field layout.
|
||||
_ if left == right => Some(left),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeNode {
|
||||
/// The kind this node's output field carries, given the kind its input carries.
|
||||
///
|
||||
@@ -466,9 +541,9 @@ impl NativeNode {
|
||||
Some(kind)
|
||||
}
|
||||
// A constant beside a vector is broadcast over it, so the operation answers with the
|
||||
// vector's own samples: a histogram times a float is still a histogram. Two vectors are
|
||||
// only one column if both sides carry the same kind; a mixed result needs two columns
|
||||
// per field, which this route does not declare.
|
||||
// vector's own samples: a histogram times a float is still a histogram. Two vectors
|
||||
// use the arithmetic result's kind, including any samples emitted by fill, and an
|
||||
// empty float field only when no possible pairing can emit a sample.
|
||||
NativeExpr::Binary {
|
||||
left,
|
||||
right,
|
||||
@@ -485,7 +560,14 @@ impl NativeNode {
|
||||
(_, NativeExpr::Scalar(_)) => kind_of(left),
|
||||
(_, _) => {
|
||||
let left_kind = kind_of(left)?;
|
||||
(left_kind == kind_of(right)?).then_some(left_kind)
|
||||
let right_kind = kind_of(right)?;
|
||||
binary_output_field_kind(
|
||||
left_kind,
|
||||
right_kind,
|
||||
matching.op,
|
||||
false,
|
||||
matching.fill,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1136,10 +1218,8 @@ pub struct NativeTree {
|
||||
impl NativeTree {
|
||||
/// Creates a tree over `root`, checking that every node's contract holds.
|
||||
///
|
||||
/// A structurally inconsistent tree is a compiler defect; a tree beyond the native depth
|
||||
/// limit is unsupported by this route and can fall back to the existing planner.
|
||||
/// A structurally inconsistent tree is a compiler defect.
|
||||
pub fn try_new(root: NativeNode) -> Result<Self> {
|
||||
ensure_native_node_depth(&root)?;
|
||||
root.validate()?;
|
||||
Ok(Self { root })
|
||||
}
|
||||
@@ -1154,7 +1234,6 @@ impl<'de> serde::Deserialize<'de> for NativeNode {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<Self, D::Error> {
|
||||
let _guard = NativeNodeDepthGuard::enter::<D::Error>()?;
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Wire {
|
||||
expr: NativeExpr,
|
||||
@@ -1177,64 +1256,31 @@ impl<'de> serde::Deserialize<'de> for NativeTree {
|
||||
root: NativeNode,
|
||||
}
|
||||
let wire = Wire::deserialize(deserializer)?;
|
||||
ensure_native_node_depth(&wire.root).map_err(serde::de::Error::custom)?;
|
||||
wire.root.validate().map_err(serde::de::Error::custom)?;
|
||||
Ok(Self { root: wire.root })
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_native_node_depth(root: &NativeNode) -> Result<()> {
|
||||
fn visit(node: &NativeNode, depth: usize) -> Result<()> {
|
||||
if depth > MAX_NATIVE_NODE_DEPTH {
|
||||
return Err(DataFusionError::NotImplemented(format!(
|
||||
"native tree nesting exceeds {MAX_NATIVE_NODE_DEPTH}"
|
||||
)));
|
||||
}
|
||||
match &node.expr {
|
||||
NativeExpr::VectorOf { input }
|
||||
| NativeExpr::ScalarOf { input }
|
||||
| NativeExpr::Instant { input, .. }
|
||||
| NativeExpr::Merge { input }
|
||||
| NativeExpr::Window { input, .. }
|
||||
| NativeExpr::Map { input, .. }
|
||||
| NativeExpr::Timestamp { input }
|
||||
| NativeExpr::Aggregate { input, .. }
|
||||
| NativeExpr::Vector { input, .. } => visit(input, depth + 1),
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
visit(input, depth + 1)?;
|
||||
visit(parameter, depth + 1)
|
||||
}
|
||||
NativeExpr::ParameterizedMap {
|
||||
input, parameters, ..
|
||||
}
|
||||
| NativeExpr::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
visit(input, depth + 1)?;
|
||||
for parameter in parameters {
|
||||
visit(parameter, depth + 1)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
NativeExpr::Binary { left, right, .. } | NativeExpr::Set { left, right, .. } => {
|
||||
visit(left, depth + 1)?;
|
||||
visit(right, depth + 1)
|
||||
}
|
||||
NativeExpr::Info { base, info, .. } => {
|
||||
visit(base, depth + 1)?;
|
||||
visit(info, depth + 1)
|
||||
}
|
||||
NativeExpr::Input(_) | NativeExpr::Scalar(_) | NativeExpr::EvalTime { .. } => Ok(()),
|
||||
}
|
||||
}
|
||||
visit(root, 1)
|
||||
}
|
||||
/// Stack one [`NativeNode::validate`] level needs below it before the check grows its stack.
|
||||
///
|
||||
/// The check recurses once per tree level, and a debug build's frame for one level is large:
|
||||
/// measured on an 80-term chain, the whole check needed about 1.9 MiB, which overflows the 2 MiB
|
||||
/// stack a test thread has. There is no depth limit on the trees this route compiles, so the check
|
||||
/// grows its stack on demand rather than relying on the caller's.
|
||||
const VALIDATE_STACK_RED_ZONE: usize = 256 * 1024;
|
||||
|
||||
/// Stack a growth adds when the reserve above is spent. It holds several levels of the same check.
|
||||
const VALIDATE_STACK_GROWTH: usize = 4 * 1024 * 1024;
|
||||
|
||||
impl NativeNode {
|
||||
/// Checks this node and its inputs against the contract of its operation.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
stacker::maybe_grow(VALIDATE_STACK_RED_ZONE, VALIDATE_STACK_GROWTH, || {
|
||||
self.validate_within_stack()
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_within_stack(&self) -> Result<()> {
|
||||
match &self.expr {
|
||||
NativeExpr::Input(input) => {
|
||||
// An input node's output is exactly the binding it was bound to: a scan hands over
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::common::{DFSchema, DataFusionError};
|
||||
use datafusion::common::DFSchema;
|
||||
use datatypes::series::SeriesOrdering;
|
||||
use datatypes::series::aggregate_state::{AggregateStateKind, StateComponent};
|
||||
use datatypes::series::samples::{FieldSpec, SeriesSchema};
|
||||
@@ -293,6 +293,214 @@ fn parameterized_topk_validates_scalar_and_grid() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binary_histogram_float_kind_matches_supported_operations() {
|
||||
use crate::extension_plan::FieldKind;
|
||||
|
||||
let histogram = input_on(grid(0, 60_000, 1));
|
||||
let float = NativeNode {
|
||||
expr: NativeExpr::Timestamp {
|
||||
input: Box::new(histogram.clone()),
|
||||
},
|
||||
output: histogram.output.clone(),
|
||||
};
|
||||
for (left, op, right, expected) in [
|
||||
(
|
||||
&histogram,
|
||||
BinaryOp::Mul,
|
||||
&float,
|
||||
Some(FieldKind::Histogram),
|
||||
),
|
||||
(
|
||||
&float,
|
||||
BinaryOp::Mul,
|
||||
&histogram,
|
||||
Some(FieldKind::Histogram),
|
||||
),
|
||||
(
|
||||
&histogram,
|
||||
BinaryOp::Div,
|
||||
&float,
|
||||
Some(FieldKind::Histogram),
|
||||
),
|
||||
(&float, BinaryOp::Div, &histogram, Some(FieldKind::Float)),
|
||||
(&histogram, BinaryOp::Add, &float, Some(FieldKind::Float)),
|
||||
(&histogram, BinaryOp::Sub, &float, Some(FieldKind::Float)),
|
||||
(
|
||||
&histogram,
|
||||
BinaryOp::Add,
|
||||
&histogram,
|
||||
Some(FieldKind::Histogram),
|
||||
),
|
||||
(
|
||||
&histogram,
|
||||
BinaryOp::Sub,
|
||||
&histogram,
|
||||
Some(FieldKind::Histogram),
|
||||
),
|
||||
(
|
||||
&histogram,
|
||||
BinaryOp::Mul,
|
||||
&histogram,
|
||||
Some(FieldKind::Float),
|
||||
),
|
||||
(
|
||||
&histogram,
|
||||
BinaryOp::Div,
|
||||
&histogram,
|
||||
Some(FieldKind::Float),
|
||||
),
|
||||
(&float, BinaryOp::Add, &float, Some(FieldKind::Float)),
|
||||
] {
|
||||
let node = NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(left.clone()),
|
||||
right: Box::new(right.clone()),
|
||||
matching: BinaryMatching { op, ..add() },
|
||||
},
|
||||
output: histogram.output.clone(),
|
||||
};
|
||||
assert_eq!(
|
||||
node.output_field_kind(FieldKind::Histogram),
|
||||
expected,
|
||||
"{op:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Arithmetic kinds cover both histogram/float directions without declining empty results.
|
||||
#[test]
|
||||
fn binary_arithmetic_field_kind_matrix() {
|
||||
use super::binary_output_field_kind;
|
||||
use crate::extension_plan::FieldKind::{Float, Histogram};
|
||||
|
||||
for op in [
|
||||
BinaryOp::Add,
|
||||
BinaryOp::Sub,
|
||||
BinaryOp::Mul,
|
||||
BinaryOp::Div,
|
||||
BinaryOp::Mod,
|
||||
BinaryOp::Pow,
|
||||
BinaryOp::Atan2,
|
||||
] {
|
||||
let histogram_pair = if matches!(op, BinaryOp::Add | BinaryOp::Sub) {
|
||||
Histogram
|
||||
} else {
|
||||
Float
|
||||
};
|
||||
let histogram_float = if matches!(op, BinaryOp::Mul | BinaryOp::Div) {
|
||||
Histogram
|
||||
} else {
|
||||
Float
|
||||
};
|
||||
let float_histogram = if op == BinaryOp::Mul {
|
||||
Histogram
|
||||
} else {
|
||||
Float
|
||||
};
|
||||
for (left, right, expected) in [
|
||||
(Float, Float, Float),
|
||||
(Histogram, Histogram, histogram_pair),
|
||||
(Histogram, Float, histogram_float),
|
||||
(Float, Histogram, float_histogram),
|
||||
] {
|
||||
assert_eq!(
|
||||
binary_output_field_kind(left, right, op, false, FillValues::default()),
|
||||
Some(expected),
|
||||
"{left:?} {op:?} {right:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fill pairs contribute their real payload kind; unsupported pairs contribute no kind.
|
||||
#[test]
|
||||
fn binary_arithmetic_field_kind_accounts_for_fill() {
|
||||
use super::binary_output_field_kind;
|
||||
use crate::extension_plan::FieldKind::{Float, Histogram};
|
||||
|
||||
let absent = FillValues::default();
|
||||
let left = FillValues {
|
||||
lhs: Some(2.0),
|
||||
rhs: None,
|
||||
};
|
||||
let right = FillValues {
|
||||
lhs: None,
|
||||
rhs: Some(2.0),
|
||||
};
|
||||
let both = FillValues {
|
||||
lhs: Some(2.0),
|
||||
rhs: Some(2.0),
|
||||
};
|
||||
for (op, fill, expected) in [
|
||||
(BinaryOp::Mul, absent, Float),
|
||||
(BinaryOp::Mul, left, Histogram),
|
||||
(BinaryOp::Mul, right, Histogram),
|
||||
(BinaryOp::Mul, both, Histogram),
|
||||
(BinaryOp::Div, absent, Float),
|
||||
(BinaryOp::Div, left, Float),
|
||||
(BinaryOp::Div, right, Histogram),
|
||||
(BinaryOp::Div, both, Histogram),
|
||||
(BinaryOp::Add, both, Histogram),
|
||||
(BinaryOp::Sub, both, Histogram),
|
||||
(BinaryOp::Mod, both, Float),
|
||||
(BinaryOp::Pow, both, Float),
|
||||
(BinaryOp::Atan2, both, Float),
|
||||
] {
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Histogram, op, false, fill),
|
||||
Some(expected),
|
||||
"{op:?} {fill:?}"
|
||||
);
|
||||
// The tree-level walk must pass the same fill on to the shared kind helper.
|
||||
let histogram = input_on(grid(0, 60_000, 1));
|
||||
let node = NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(histogram.clone()),
|
||||
right: Box::new(histogram.clone()),
|
||||
matching: BinaryMatching { op, fill, ..add() },
|
||||
},
|
||||
output: histogram.output.clone(),
|
||||
};
|
||||
assert_eq!(
|
||||
node.output_field_kind(Histogram),
|
||||
Some(expected),
|
||||
"{op:?} {fill:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// The real mixed-kind operation and a fill may emit different kinds in one field. Neither
|
||||
// silently claims homogeneity; mixed no-fill composition and comparisons keep their old kinds.
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Float, BinaryOp::Mul, false, left),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Float, Histogram, BinaryOp::Mul, false, right),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Float, BinaryOp::Add, false, left),
|
||||
Some(Float)
|
||||
);
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Float, BinaryOp::Mul, false, absent),
|
||||
Some(Histogram)
|
||||
);
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Histogram, BinaryOp::Eq, false, both),
|
||||
Some(Histogram)
|
||||
);
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Float, BinaryOp::Eq, false, both),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
binary_output_field_kind(Histogram, Float, BinaryOp::Eq, true, both),
|
||||
Some(Float)
|
||||
);
|
||||
}
|
||||
|
||||
/// A compiled tree is accepted when every node's contract holds.
|
||||
#[test]
|
||||
fn a_window_over_raw_samples_is_valid() {
|
||||
@@ -2044,9 +2252,7 @@ fn info_still_refuses_an_evaluated_value_vector() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tree_depth_limit_holds_across_lifecycle_on_named_worker_stack() {
|
||||
use super::MAX_NATIVE_NODE_DEPTH;
|
||||
|
||||
fn native_tree_deeper_than_64_round_trips_and_validates() {
|
||||
fn nested_maps(map_count: usize) -> NativeTree {
|
||||
let mut node = input_on(grid(0, 60_000, 1));
|
||||
let output = node.output.clone();
|
||||
@@ -2064,100 +2270,20 @@ fn native_tree_depth_limit_holds_across_lifecycle_on_named_worker_stack() {
|
||||
NativeTree { root: node }
|
||||
}
|
||||
|
||||
// A leaf counts as one node; test exactly the cap and rejection on a named 2 MiB worker.
|
||||
std::thread::Builder::new()
|
||||
.name("native-tree-depth-boundary".to_string())
|
||||
.stack_size(2 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let at_limit = nested_maps(MAX_NATIVE_NODE_DEPTH - 1);
|
||||
let checked = NativeTree::try_new(at_limit.root.clone()).unwrap();
|
||||
let bytes = postcard::to_stdvec(&checked).unwrap();
|
||||
let decoded: NativeTree = postcard::from_bytes(&bytes).unwrap();
|
||||
decoded.root().validate().unwrap();
|
||||
drop((decoded, bytes, checked, at_limit));
|
||||
// This tests a representative depth, not a guarantee about the available worker stack.
|
||||
let deep = nested_maps(80);
|
||||
let checked = NativeTree::try_new(deep.root.clone()).unwrap();
|
||||
let bytes = postcard::to_stdvec(&checked).unwrap();
|
||||
let decoded: NativeTree = postcard::from_bytes(&bytes).unwrap();
|
||||
assert_eq!(checked, decoded);
|
||||
decoded.root().validate().unwrap();
|
||||
|
||||
// Construct wire data beyond the cap without passing through try_new.
|
||||
let over_limit = nested_maps(MAX_NATIVE_NODE_DEPTH);
|
||||
let bytes = postcard::to_stdvec(&over_limit).unwrap();
|
||||
assert_eq!(
|
||||
postcard::from_bytes::<NativeTree>(&bytes).unwrap_err(),
|
||||
postcard::Error::SerdeDeCustom
|
||||
);
|
||||
assert!(matches!(
|
||||
NativeTree::try_new(over_limit.root.clone()),
|
||||
Err(DataFusionError::NotImplemented(_))
|
||||
));
|
||||
drop(over_limit);
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tree_depth_limit_checks_dynamic_children() {
|
||||
use super::{MAX_NATIVE_NODE_DEPTH, ParameterizedOp, ensure_native_node_depth};
|
||||
use crate::histogram_eval::HistogramFunction;
|
||||
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let mut deep = input_on(Arc::clone(&grid));
|
||||
for _ in 0..MAX_NATIVE_NODE_DEPTH - 1 {
|
||||
deep = map_over(deep);
|
||||
}
|
||||
let scalar_of = |input| NativeNode {
|
||||
expr: NativeExpr::ScalarOf {
|
||||
input: Box::new(input),
|
||||
},
|
||||
output: NativeType::Scalar(Arc::clone(&grid)),
|
||||
};
|
||||
let scalar = scalar_on(1.0, Arc::clone(&grid));
|
||||
let input = input_on(Arc::clone(&grid));
|
||||
let topk = |input: NativeNode, parameter| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
},
|
||||
};
|
||||
let histogram = |input: NativeNode, parameters| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Fraction {
|
||||
lower: 0.0,
|
||||
upper: 0.0,
|
||||
}),
|
||||
parameters,
|
||||
},
|
||||
};
|
||||
let map = |input: NativeNode, parameters| NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: NativeExpr::ParameterizedMap {
|
||||
input: Box::new(input),
|
||||
op: ParameterizedOp::Clamp,
|
||||
parameters,
|
||||
},
|
||||
};
|
||||
assert!(ensure_native_node_depth(&deep).is_ok());
|
||||
for root in [
|
||||
scalar_of(deep.clone()),
|
||||
topk(deep.clone(), scalar.clone()),
|
||||
topk(input.clone(), scalar_of(deep.clone())),
|
||||
histogram(deep.clone(), vec![scalar.clone(), scalar.clone()]),
|
||||
histogram(input.clone(), vec![scalar.clone(), scalar_of(deep.clone())]),
|
||||
map(deep.clone(), vec![scalar.clone(), scalar.clone()]),
|
||||
map(input, vec![scalar, scalar_of(deep)]),
|
||||
] {
|
||||
assert!(matches!(
|
||||
NativeTree::try_new(root),
|
||||
Err(DataFusionError::NotImplemented(_))
|
||||
));
|
||||
}
|
||||
// Decoding must still validate the tree's structural contract.
|
||||
let mut invalid = nested_maps(80);
|
||||
invalid.root.output = NativeType::Scalar(grid(0, 60_000, 1));
|
||||
assert!(NativeTree::try_new(invalid.root.clone()).is_err());
|
||||
let bytes = postcard::to_stdvec(&invalid).unwrap();
|
||||
assert!(postcard::from_bytes::<NativeTree>(&bytes).is_err());
|
||||
}
|
||||
|
||||
/// The index postcard writes for a [`NativeExpr`] variant is a wire contract, not an implementation
|
||||
|
||||
@@ -24,6 +24,7 @@ use std::collections::hash_map::Entry;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use common_query::promql_annotations::PromqlAnnotationCollector;
|
||||
use datafusion::error::{DataFusionError, Result};
|
||||
use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
|
||||
use datatypes::series::histogram::{CounterResetHint, HistogramBatchBuilder, HistogramView};
|
||||
@@ -33,6 +34,7 @@ use datatypes::series::samples::{
|
||||
use datatypes::series::time::{EvalGrid, GridRunPair, GridSpan, TimeAxis};
|
||||
use datatypes::series::{Label, SeriesLabels};
|
||||
|
||||
use crate::functions::{NativeHistogramAdd, NativeHistogramSub};
|
||||
use crate::grid_series::{GridSample, GridSeriesBuilder, METADATA_LABELS, external, owned};
|
||||
use crate::histogram_eval::float_histogram::FloatHistogram;
|
||||
|
||||
@@ -201,7 +203,7 @@ pub fn binary_vectors(
|
||||
rhs_schema: &SeriesSchema,
|
||||
rhs: &[Series],
|
||||
) -> Result<Vec<Series>> {
|
||||
binary_vectors_inner(matching, lhs_schema, lhs, rhs_schema, rhs, None, None)
|
||||
binary_vectors_inner(matching, lhs_schema, lhs, rhs_schema, rhs, None, None, None)
|
||||
}
|
||||
|
||||
/// [`binary_vectors`], handing the live workspace it holds to `reserve` as it allocates.
|
||||
@@ -225,6 +227,7 @@ struct FieldEvidence<'a> {
|
||||
evidence: &'a DuplicateEvidence,
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn binary_vectors_with_reservation(
|
||||
matching: &BinaryMatching,
|
||||
lhs_schema: &SeriesSchema,
|
||||
@@ -232,6 +235,7 @@ pub(crate) fn binary_vectors_with_reservation(
|
||||
rhs_schema: &SeriesSchema,
|
||||
rhs: &[Series],
|
||||
evidence: Option<NodeEvidence<'_>>,
|
||||
annotations: Option<&PromqlAnnotationCollector>,
|
||||
reserve: &mut dyn FnMut(usize) -> Result<()>,
|
||||
) -> Result<Vec<Series>> {
|
||||
let mut budget = Budget::new(reserve);
|
||||
@@ -242,12 +246,14 @@ pub(crate) fn binary_vectors_with_reservation(
|
||||
rhs_schema,
|
||||
rhs,
|
||||
evidence,
|
||||
annotations,
|
||||
Some(&mut budget),
|
||||
)
|
||||
}
|
||||
|
||||
/// The one binary kernel: one walk, one lookup and one error order, whether or not a caller
|
||||
/// accounts for the workspace. Every figure is reserved only when `budget` is held.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn binary_vectors_inner(
|
||||
matching: &BinaryMatching,
|
||||
lhs_schema: &SeriesSchema,
|
||||
@@ -255,6 +261,7 @@ fn binary_vectors_inner(
|
||||
rhs_schema: &SeriesSchema,
|
||||
rhs: &[Series],
|
||||
evidence: Option<NodeEvidence<'_>>,
|
||||
annotations: Option<&PromqlAnnotationCollector>,
|
||||
mut budget: Option<&mut Budget<'_>>,
|
||||
) -> Result<Vec<Series>> {
|
||||
// The grid both operands are evaluated on, checked before anything is reserved for them.
|
||||
@@ -284,6 +291,7 @@ fn binary_vectors_inner(
|
||||
&rhs,
|
||||
&mut errors,
|
||||
field_evidence,
|
||||
annotations,
|
||||
budget.as_deref_mut(),
|
||||
)?;
|
||||
// The field's flags, tables and sort are destroyed with its frame, so its phase ends here:
|
||||
@@ -734,6 +742,7 @@ fn histogram_phase_bytes(many: &Operand<'_>, one: &Operand<'_>) -> usize {
|
||||
/// The field's tables and the match set it builds are its own workspace: they replace what the field
|
||||
/// before held and are gone when it returns, which is why the caller admits that release after the
|
||||
/// call. The messages it appended and the rows it filed are what stays live.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn binary_field(
|
||||
matching: &BinaryMatching,
|
||||
grid: &EvalGrid,
|
||||
@@ -741,6 +750,7 @@ fn binary_field(
|
||||
rhs: &Operand<'_>,
|
||||
errors: &mut Vec<(usize, u8, String)>,
|
||||
evidence: Option<FieldEvidence<'_>>,
|
||||
annotations: Option<&PromqlAnnotationCollector>,
|
||||
mut budget: Option<&mut Budget<'_>>,
|
||||
) -> Result<Results> {
|
||||
let (many, one, include, one_is_left) = match &matching.cardinality {
|
||||
@@ -845,6 +855,7 @@ fn binary_field(
|
||||
let mut pairs = Pairs {
|
||||
matching,
|
||||
one_is_left,
|
||||
annotations,
|
||||
group_keys: Vec::new(),
|
||||
group_ids: HashMap::new(),
|
||||
// One-to-one: a match group may be matched once per point; Prometheus counts a filtered
|
||||
@@ -913,6 +924,7 @@ fn binary_field(
|
||||
point,
|
||||
many_value,
|
||||
one_value,
|
||||
true,
|
||||
budget.as_deref_mut(),
|
||||
)?;
|
||||
}
|
||||
@@ -955,6 +967,7 @@ fn binary_field(
|
||||
point,
|
||||
sample,
|
||||
SampleRef::Float(value),
|
||||
false,
|
||||
budget.as_deref_mut(),
|
||||
)?;
|
||||
}
|
||||
@@ -999,6 +1012,7 @@ fn binary_field(
|
||||
point,
|
||||
SampleRef::Float(value),
|
||||
sample,
|
||||
false,
|
||||
budget.as_deref_mut(),
|
||||
)?;
|
||||
}
|
||||
@@ -1085,6 +1099,7 @@ fn binary_field(
|
||||
struct Pairs<'a> {
|
||||
matching: &'a BinaryMatching,
|
||||
one_is_left: bool,
|
||||
annotations: Option<&'a PromqlAnnotationCollector>,
|
||||
group_keys: Vec<GroupKey>,
|
||||
group_ids: HashMap<GroupKey, usize>,
|
||||
/// The match groups an operation matched at a point, kept where Prometheus consults them:
|
||||
@@ -1125,6 +1140,7 @@ impl Pairs<'_> {
|
||||
point: usize,
|
||||
many_value: SampleRef<'_>,
|
||||
one_value: SampleRef<'_>,
|
||||
actual_pair: bool,
|
||||
mut budget: Option<&mut Budget<'_>>,
|
||||
) -> Result<RowSlot<'s>> {
|
||||
if matches!(many_value, SampleRef::Missing) || matches!(one_value, SampleRef::Missing) {
|
||||
@@ -1135,7 +1151,27 @@ impl Pairs<'_> {
|
||||
} else {
|
||||
(many_value, one_value)
|
||||
};
|
||||
let Some((result, keep)) = elem_binop(self.matching.op, lhs_value, rhs_value)? else {
|
||||
// Prometheus runs the operation on the float a fill value puts in place of a missing sample,
|
||||
// so a filled pair reports the legacy dropped-sample Info too. What stays out is not the
|
||||
// fill but the pair: one missing a side never reaches here, and a supported operation that
|
||||
// declines for another reason -- incompatible bucket layouts -- is already outside what
|
||||
// [`unsupported_histogram_arithmetic`] reports.
|
||||
let dropped_op = if self.annotations.is_some() {
|
||||
unsupported_histogram_arithmetic(self.matching.op, &lhs_value, &rhs_value)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// A fill value is a float, so a filled pair never holds the two histograms `elem_binop`
|
||||
// states its reconciliation and reset diagnostics for: those stay off for it.
|
||||
let pair_annotations = if actual_pair { self.annotations } else { None };
|
||||
let Some((result, keep)) =
|
||||
elem_binop(self.matching.op, lhs_value, rhs_value, pair_annotations)?
|
||||
else {
|
||||
if let (Some(collector), Some(op)) = (self.annotations, dropped_op) {
|
||||
collector.record_info(format!(
|
||||
"{op}: dropped native histogram samples because this binary operation is not supported for native histograms"
|
||||
));
|
||||
}
|
||||
// The operator skipped the pair and nothing of it was filed.
|
||||
return Ok(slot);
|
||||
};
|
||||
@@ -2156,14 +2192,45 @@ impl<'a> Operand<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The legacy Info applies only to unsupported histogram arithmetic, not to other reasons a
|
||||
/// supported histogram operation can decline a pair (such as incompatible bucket layouts).
|
||||
fn unsupported_histogram_arithmetic(
|
||||
op: BinaryOp,
|
||||
lhs: &SampleRef<'_>,
|
||||
rhs: &SampleRef<'_>,
|
||||
) -> Option<&'static str> {
|
||||
let symbol = match op {
|
||||
BinaryOp::Add => "+",
|
||||
BinaryOp::Sub => "-",
|
||||
BinaryOp::Mul => "*",
|
||||
BinaryOp::Div => "/",
|
||||
BinaryOp::Mod => "%",
|
||||
BinaryOp::Pow => "^",
|
||||
BinaryOp::Atan2 => "atan2",
|
||||
_ => return None,
|
||||
};
|
||||
let unsupported = match (lhs, rhs) {
|
||||
(SampleRef::Histogram(_), SampleRef::Histogram(_)) => {
|
||||
!matches!(op, BinaryOp::Add | BinaryOp::Sub)
|
||||
}
|
||||
(SampleRef::Histogram(_), SampleRef::Float(_)) => {
|
||||
!matches!(op, BinaryOp::Mul | BinaryOp::Div)
|
||||
}
|
||||
(SampleRef::Float(_), SampleRef::Histogram(_)) => op != BinaryOp::Mul,
|
||||
_ => false,
|
||||
};
|
||||
unsupported.then_some(symbol)
|
||||
}
|
||||
|
||||
/// Prometheus `vectorElemBinop` over present samples: the value and whether to keep it, or
|
||||
/// `None` when Prometheus reports the pair as an annotation and skips it without counting the
|
||||
/// match: an operator the sample types do not support, or an addition or subtraction of
|
||||
/// histograms that mix exponential and custom buckets or whose counter reset hints collide.
|
||||
/// histograms that mix exponential and custom buckets.
|
||||
fn elem_binop(
|
||||
op: BinaryOp,
|
||||
lhs: SampleRef<'_>,
|
||||
rhs: SampleRef<'_>,
|
||||
annotations: Option<&PromqlAnnotationCollector>,
|
||||
) -> Result<Option<(GridSample, bool)>> {
|
||||
let histogram = |view: &HistogramView<'_>| FloatHistogram::try_from_view(view);
|
||||
let mut result = match (lhs, rhs) {
|
||||
@@ -2192,16 +2259,50 @@ fn elem_binop(
|
||||
return Ok(Some((GridSample::Histogram(Box::new(lhs)), keep)));
|
||||
}
|
||||
BinaryOp::Add | BinaryOp::Sub => {
|
||||
let name = if op == BinaryOp::Add {
|
||||
NativeHistogramAdd::name()
|
||||
} else {
|
||||
NativeHistogramSub::name()
|
||||
};
|
||||
// The layouts decide whether there is a pair at all: Prometheus reports a pair
|
||||
// whose bucket layouts cannot be combined as that one error, before it looks at
|
||||
// the reset hints, so a pair that fails both carries only the schema warning.
|
||||
// Synthetic fill samples have no collector here.
|
||||
if lhs.uses_custom_buckets() != rhs.uses_custom_buckets() {
|
||||
if let Some(collector) = annotations {
|
||||
collector.record_warning(format!(
|
||||
"{name}: dropped native histogram sample with incompatible schemas"
|
||||
));
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
if let Some(collector) = annotations
|
||||
&& matches!(
|
||||
(lhs.reset_hint, rhs.reset_hint),
|
||||
(
|
||||
CounterResetHint::CounterReset,
|
||||
CounterResetHint::NotCounterReset
|
||||
) | (
|
||||
CounterResetHint::NotCounterReset,
|
||||
CounterResetHint::CounterReset
|
||||
)
|
||||
)
|
||||
{
|
||||
collector.record_warning(format!(
|
||||
"{name}: native histogram counter reset hints contradict"
|
||||
));
|
||||
}
|
||||
let outcome = if op == BinaryOp::Add {
|
||||
lhs.add(&rhs)?
|
||||
} else {
|
||||
lhs.sub(&rhs)?
|
||||
};
|
||||
if outcome.counter_reset_collision {
|
||||
return Ok(None);
|
||||
if outcome.custom_bounds_reconciled
|
||||
&& let Some(collector) = annotations
|
||||
{
|
||||
collector.record_info(format!(
|
||||
"{name}: reconciled native histograms with different custom buckets"
|
||||
));
|
||||
}
|
||||
if op == BinaryOp::Sub {
|
||||
lhs.reset_hint = CounterResetHint::Gauge;
|
||||
|
||||
@@ -1154,11 +1154,9 @@ fn histogram_operators_match_prometheus() {
|
||||
);
|
||||
}
|
||||
|
||||
/// Prometheus skips a histogram `+` or `-` whose counter reset hints collide, before counting
|
||||
/// the match: the colliding pair gives nothing and does not make the other pair of its group a
|
||||
/// second match.
|
||||
/// Contradictory reset hints retain the value and therefore count as a successful match.
|
||||
#[test]
|
||||
fn colliding_reset_hints_skip_the_pair() {
|
||||
fn colliding_reset_hints_still_count_as_a_match() {
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 10_000, 1).unwrap()).unwrap());
|
||||
let schema = Arc::new(SeriesSchema {
|
||||
fields: vec![FieldSpec {
|
||||
@@ -1182,7 +1180,6 @@ fn colliding_reset_hints_skip_the_pair() {
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
// Hints: 0 unknown, 1 counter reset, 2 not a counter reset.
|
||||
let lhs = [
|
||||
series(&[("__name__", "a"), ("instance", "1"), ("job", "x")], 1),
|
||||
series(&[("__name__", "a"), ("instance", "2"), ("job", "x")], 0),
|
||||
@@ -1196,8 +1193,13 @@ fn colliding_reset_hints_skip_the_pair() {
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill: FillValues::default(),
|
||||
};
|
||||
let result = binary_vectors(&matching, &schema, &lhs, &schema, &rhs).unwrap();
|
||||
assert_eq!(result.len(), 1, "{op:?}");
|
||||
let error = binary_vectors(&matching, &schema, &lhs, &schema, &rhs)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
error.contains("many-to-one matching must be explicit"),
|
||||
"{op:?}: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1254,6 +1256,268 @@ fn a_failed_pair_leaves_its_sample_to_the_fill() {
|
||||
));
|
||||
}
|
||||
|
||||
/// A pair a fill value made is a pair the operation is run on, so an unsupported histogram pairing
|
||||
/// it makes reports the same dropped-samples Info a matched pair does.
|
||||
///
|
||||
/// Prometheus fills a missing side and then runs `doBinOp` on the pair it formed, so a histogram
|
||||
/// filled with a float meets the unsupported pairing `histogram + float` at every point and carries
|
||||
/// the Info a matched pair would. What must not carry it is a fill that answers with a sample the
|
||||
/// operation supports, and a point whose pair is never formed at all.
|
||||
#[test]
|
||||
fn a_filled_pair_reports_the_dropped_histogram_info() {
|
||||
use common_query::promql_annotations::PromqlAnnotationCollector;
|
||||
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 10_000, 2).unwrap()).unwrap());
|
||||
let schema = Arc::new(schema(1, &grid));
|
||||
let labels = |name: &str, job: &str| -> Labels {
|
||||
Labels::from([
|
||||
("__name__".to_string(), name.to_string()),
|
||||
("job".to_string(), job.to_string()),
|
||||
])
|
||||
};
|
||||
// count 5, sum 5: one span of one bucket holding 5.
|
||||
let text = "0/0/0/4014000000000000/4014000000000000/0/0_1/4014000000000000///";
|
||||
let histograms = |job: &str| {
|
||||
mixed_series(
|
||||
&schema,
|
||||
&labels("hist", job),
|
||||
&BTreeMap::new(),
|
||||
&BTreeMap::from([(0, text.to_string()), (1, text.to_string())]),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let floats = |job: &str| {
|
||||
mixed_series(
|
||||
&schema,
|
||||
&labels("float", job),
|
||||
&BTreeMap::from([(0, 2.0f64.to_bits()), (1, 2.0f64.to_bits())]),
|
||||
&BTreeMap::new(),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
// The two series carry different `job` labels, so no point of either is matched; with a fill
|
||||
// value every point of each is paired with the float the modifier names instead.
|
||||
let diagnostics = |matching: &BinaryMatching,
|
||||
lhs: &[Series],
|
||||
rhs: &[Series]|
|
||||
-> (usize, Vec<String>, Vec<String>) {
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let output = binary_vectors_with_reservation(
|
||||
matching,
|
||||
&schema,
|
||||
lhs,
|
||||
&schema,
|
||||
rhs,
|
||||
None,
|
||||
Some(&collector),
|
||||
&mut |_| Ok(()),
|
||||
)
|
||||
.unwrap();
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
(output.len(), warnings, infos)
|
||||
};
|
||||
let matching = |op: BinaryOp, fill: FillValues| BinaryMatching {
|
||||
op,
|
||||
return_bool: false,
|
||||
labels: MatchingLabels::On(vec![Arc::from("job")]),
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill,
|
||||
};
|
||||
let dropped = |symbol: &str| {
|
||||
vec![format!(
|
||||
"{symbol}: dropped native histogram samples because this binary operation is not supported for native histograms"
|
||||
)]
|
||||
};
|
||||
// `fill_right` answers a missing right-hand sample, so the histogram is the left one and each
|
||||
// point is the pair `histogram + 0`.
|
||||
assert_eq!(
|
||||
diagnostics(
|
||||
&matching(
|
||||
BinaryOp::Add,
|
||||
FillValues {
|
||||
lhs: None,
|
||||
rhs: Some(0.0)
|
||||
}
|
||||
),
|
||||
&[histograms("x")],
|
||||
&[floats("y")],
|
||||
),
|
||||
(0, Vec::new(), dropped("+")),
|
||||
);
|
||||
// `fill_left` answers a missing left-hand one, so the histogram is the right one and the pair
|
||||
// is `0 + histogram`.
|
||||
assert_eq!(
|
||||
diagnostics(
|
||||
&matching(
|
||||
BinaryOp::Add,
|
||||
FillValues {
|
||||
lhs: Some(0.0),
|
||||
rhs: None
|
||||
}
|
||||
),
|
||||
&[floats("y")],
|
||||
&[histograms("x")],
|
||||
),
|
||||
(0, Vec::new(), dropped("+")),
|
||||
);
|
||||
// A fill the operation supports answers a sample: nothing was dropped, so nothing is reported.
|
||||
assert_eq!(
|
||||
diagnostics(
|
||||
&matching(
|
||||
BinaryOp::Mul,
|
||||
FillValues {
|
||||
lhs: None,
|
||||
rhs: Some(2.0)
|
||||
}
|
||||
),
|
||||
&[histograms("x")],
|
||||
&[floats("y")],
|
||||
),
|
||||
(1, Vec::new(), Vec::new()),
|
||||
);
|
||||
// With no fill value the two sides never pair, so no sample exists to be dropped.
|
||||
assert_eq!(
|
||||
diagnostics(
|
||||
&matching(BinaryOp::Add, FillValues::default()),
|
||||
&[histograms("x")],
|
||||
&[floats("y")],
|
||||
),
|
||||
(0, Vec::new(), Vec::new()),
|
||||
);
|
||||
}
|
||||
|
||||
/// A pair whose bucket layouts cannot be combined carries that one error, and no reset-collision
|
||||
/// warning even when its hints contradict.
|
||||
///
|
||||
/// Prometheus reports the incompatible layouts before it reaches the hints, so a pair that fails
|
||||
/// both is reported once. Contradictory hints still earn their warning when the layouts can be
|
||||
/// combined: the check order decides which diagnostic a pair that fails both carries, not whether
|
||||
/// the collision is ever reported.
|
||||
#[test]
|
||||
fn an_incompatible_bucket_layout_drops_the_reset_collision_warning() {
|
||||
use common_query::promql_annotations::PromqlAnnotationCollector;
|
||||
use datatypes::series::histogram::CounterResetHint;
|
||||
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 10_000, 1).unwrap()).unwrap());
|
||||
let schema = Arc::new(schema(1, &grid));
|
||||
let labels = |job: &str| -> Labels {
|
||||
Labels::from([
|
||||
("__name__".to_string(), "hist".to_string()),
|
||||
("job".to_string(), job.to_string()),
|
||||
])
|
||||
};
|
||||
let operand = |job: &str, text: &str, hint: CounterResetHint| -> Series {
|
||||
let histogram = OwnedHistogram::parse(text);
|
||||
let mut read = FloatHistogram::try_from_view(&histogram.view()).unwrap();
|
||||
read.reset_hint = hint;
|
||||
let mut builder = GridSeriesBuilder::try_new(schema.clone()).unwrap();
|
||||
builder
|
||||
.push_samples(0, &[Some(GridSample::Histogram(Box::new(read)))])
|
||||
.unwrap();
|
||||
builder.finish(metric(&labels(job))).unwrap().unwrap()
|
||||
};
|
||||
// count 5, sum 5: one span of one bucket holding 5, no custom bounds.
|
||||
let exponential = "0/0/0/4014000000000000/4014000000000000/0/0_1/4014000000000000///";
|
||||
// count 2, sum 3: one span of one bucket holding 2, over the custom upper bounds given.
|
||||
let custom = |bounds: &str| {
|
||||
format!("-53/0/0/4000000000000000/4008000000000000/0/0_1/4000000000000000///{bounds}")
|
||||
};
|
||||
// Both operands carry the same `job`, so they match on it and the operation is applied to the
|
||||
// pair; what it reports is what is under test.
|
||||
let diagnostics = |lhs: &Series, rhs: &Series| -> (usize, Vec<String>, Vec<String>) {
|
||||
let matching = BinaryMatching {
|
||||
op: BinaryOp::Add,
|
||||
return_bool: false,
|
||||
labels: MatchingLabels::On(vec![Arc::from("job")]),
|
||||
cardinality: Cardinality::OneToOne,
|
||||
fill: FillValues::default(),
|
||||
};
|
||||
let collector = PromqlAnnotationCollector::default();
|
||||
let output = binary_vectors_with_reservation(
|
||||
&matching,
|
||||
&schema,
|
||||
std::slice::from_ref(lhs),
|
||||
&schema,
|
||||
std::slice::from_ref(rhs),
|
||||
None,
|
||||
Some(&collector),
|
||||
&mut |_| Ok(()),
|
||||
)
|
||||
.unwrap();
|
||||
let mut warnings = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
collector.append_to(&mut warnings, &mut infos);
|
||||
(output.len(), warnings, infos)
|
||||
};
|
||||
let schema_warning = vec![
|
||||
"prom_native_histogram_add: dropped native histogram sample with incompatible schemas"
|
||||
.to_string(),
|
||||
];
|
||||
let collision = vec![
|
||||
"prom_native_histogram_add: native histogram counter reset hints contradict".to_string(),
|
||||
];
|
||||
let reconciled = vec![
|
||||
"prom_native_histogram_add: reconciled native histograms with different custom buckets"
|
||||
.to_string(),
|
||||
];
|
||||
|
||||
// One exponential layout and one custom one, in either order: the layouts cannot be combined,
|
||||
// so the pair is answered with the schema warning and never reaches the hints.
|
||||
let one_bound = custom("3ff0000000000000");
|
||||
for (lhs, rhs, hints) in [
|
||||
(
|
||||
exponential,
|
||||
one_bound.as_str(),
|
||||
(
|
||||
CounterResetHint::CounterReset,
|
||||
CounterResetHint::NotCounterReset,
|
||||
),
|
||||
),
|
||||
(
|
||||
one_bound.as_str(),
|
||||
exponential,
|
||||
(
|
||||
CounterResetHint::NotCounterReset,
|
||||
CounterResetHint::CounterReset,
|
||||
),
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
diagnostics(&operand("a", lhs, hints.0), &operand("a", rhs, hints.1)),
|
||||
(0, schema_warning.clone(), Vec::new()),
|
||||
"{lhs} + {rhs}",
|
||||
);
|
||||
}
|
||||
|
||||
// The same contradiction over one exponential layout, and over two custom ones the operation
|
||||
// reconciles: there the collision is reported, and only the layouts that cannot be combined
|
||||
// silence it.
|
||||
assert_eq!(
|
||||
diagnostics(
|
||||
&operand("a", exponential, CounterResetHint::CounterReset),
|
||||
&operand("a", exponential, CounterResetHint::NotCounterReset),
|
||||
),
|
||||
(1, collision.clone(), Vec::new()),
|
||||
);
|
||||
assert_eq!(
|
||||
diagnostics(
|
||||
&operand(
|
||||
"a",
|
||||
&custom("3ff0000000000000"),
|
||||
CounterResetHint::CounterReset
|
||||
),
|
||||
&operand(
|
||||
"a",
|
||||
&custom("3ff0000000000000,4000000000000000"),
|
||||
CounterResetHint::NotCounterReset,
|
||||
),
|
||||
),
|
||||
(1, collision, reconciled),
|
||||
);
|
||||
}
|
||||
|
||||
/// A binary pairing walks two operands of many single-position runs and still reports every
|
||||
/// position exactly once.
|
||||
///
|
||||
@@ -1630,6 +1894,7 @@ fn budgeted(
|
||||
rhs_schema,
|
||||
rhs,
|
||||
None,
|
||||
None,
|
||||
&mut |total| {
|
||||
if total > limit {
|
||||
return Err(DataFusionError::ResourcesExhausted(format!(
|
||||
@@ -3238,6 +3503,7 @@ fn a_histogram_comparison_filter_keeps_the_payload_of_the_operand_it_keeps() {
|
||||
&schema,
|
||||
&rhs,
|
||||
None,
|
||||
None,
|
||||
&mut |total| {
|
||||
reserved.push(total);
|
||||
Ok(())
|
||||
@@ -3574,6 +3840,7 @@ fn binary_field_publishes_presence_and_records_excluded_duplicate() {
|
||||
evidence: &evidence,
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
|
||||
@@ -653,21 +653,7 @@ fn binary_operation(context: &Context, binary: &BinaryExpr) -> Result<NativeNode
|
||||
{
|
||||
let left = compile_expr(context, &binary.lhs)?;
|
||||
let right = compile_expr(context, &binary.rhs)?;
|
||||
let (Some(lhs), Some(rhs)) = (left.output.samples(), right.output.samples()) else {
|
||||
return Err(DataFusionError::Internal(format!(
|
||||
"the binary operation `{binary}` compiled a vector operand to something other than \
|
||||
samples: a vector-typed expression compiles to samples, so the two disagree"
|
||||
)));
|
||||
};
|
||||
let output = NativeType::Samples(binary_output_schema(lhs, rhs)?);
|
||||
return Ok(NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(left),
|
||||
right: Box::new(right),
|
||||
matching: vector_matching(binary, op)?,
|
||||
},
|
||||
output,
|
||||
});
|
||||
return bind_vector_binary(binary, left, right, op);
|
||||
}
|
||||
// What is left has a constant on one side, and a modifier is refused rather than dropped. This
|
||||
// is deliberately narrower than Prometheus, which drops an `on` or `ignoring` written on a
|
||||
@@ -707,6 +693,44 @@ fn binary_operation(context: &Context, binary: &BinaryExpr) -> Result<NativeNode
|
||||
))
|
||||
}
|
||||
|
||||
/// Binds the same vector operation over compiled or already evaluated operands.
|
||||
fn bind_vector_binary(
|
||||
binary: &BinaryExpr,
|
||||
left: NativeNode,
|
||||
right: NativeNode,
|
||||
op: BinaryOp,
|
||||
) -> Result<NativeNode> {
|
||||
let (Some(lhs), Some(rhs)) = (left.output.samples(), right.output.samples()) else {
|
||||
return Err(DataFusionError::Internal(format!(
|
||||
"the binary operation `{binary}` compiled a vector operand to something other than samples"
|
||||
)));
|
||||
};
|
||||
let output = NativeType::Samples(binary_output_schema(lhs, rhs)?);
|
||||
Ok(NativeNode {
|
||||
expr: NativeExpr::Binary {
|
||||
left: Box::new(left),
|
||||
right: Box::new(right),
|
||||
matching: vector_matching(binary, op)?,
|
||||
},
|
||||
output,
|
||||
})
|
||||
}
|
||||
|
||||
/// Uses the ordinary binary binder for two evaluated frame inputs.
|
||||
pub(crate) fn bind_evaluated_binary(
|
||||
binary: &BinaryExpr,
|
||||
left: NativeNode,
|
||||
right: NativeNode,
|
||||
) -> Result<NativeTree> {
|
||||
let op = binary_op(&binary.op).ok_or_else(|| {
|
||||
not_implemented(&format!(
|
||||
"the binary operator `{}` in `{binary}`",
|
||||
binary.op
|
||||
))
|
||||
})?;
|
||||
NativeTree::try_new(bind_vector_binary(binary, left, right, op)?)
|
||||
}
|
||||
|
||||
/// Compiles `and`, `or` or `unless` between two vectors, matched by the labels the query wrote.
|
||||
///
|
||||
/// The parser already refuses a scalar operand, `bool` and `group_left` / `group_right` on a set
|
||||
|
||||
@@ -1187,6 +1187,40 @@ impl PromPlanner {
|
||||
);
|
||||
}
|
||||
|
||||
// The same whole native binary the statement root was offered can sit below a
|
||||
// legacy-only wrapper. A nested retry is valid only on the statement's own grid.
|
||||
if self.native_fused
|
||||
&& self.ctx.start == self.ctx.stmt_start
|
||||
&& self.ctx.end == self.ctx.stmt_end
|
||||
{
|
||||
let saved = self.ctx.clone();
|
||||
let native = self
|
||||
.try_plan_native_fused(&PromExpr::Binary(binary_expr.clone()), query_engine_state)
|
||||
.await;
|
||||
self.ctx = saved;
|
||||
if let Some(plan) = native? {
|
||||
let mut fields = plan.schema().fields().iter();
|
||||
if let Some(time) = fields.next().map(|field| field.name().clone()) {
|
||||
let values = fields
|
||||
.clone()
|
||||
.filter(|field| field.data_type() != &ArrowDataType::Utf8)
|
||||
.map(|field| field.name().clone())
|
||||
.collect();
|
||||
let tags = fields
|
||||
.filter(|field| field.data_type() == &ArrowDataType::Utf8)
|
||||
.map(|field| field.name().clone())
|
||||
.collect();
|
||||
self.ctx.reset();
|
||||
self.ctx.reset_table_name_and_schema();
|
||||
self.ctx.aggregation_field_labels.clear();
|
||||
self.ctx.time_index_column = Some(time);
|
||||
self.ctx.field_columns = values;
|
||||
self.ctx.tag_columns = tags;
|
||||
return Ok(plan);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(plan) = self.try_plan_binary_island(binary_expr).await? {
|
||||
return Ok(plan);
|
||||
}
|
||||
@@ -1377,32 +1411,14 @@ impl PromPlanner {
|
||||
}
|
||||
// both are columns. join them on time index
|
||||
(None, None) => {
|
||||
let native_pair =
|
||||
self.native_fused && native::hybrid_binary_admitted(*op, modifier, lhs, rhs);
|
||||
let native = if native_pair {
|
||||
match (
|
||||
self.try_plan_native_operand(lhs, query_engine_state)
|
||||
.await?,
|
||||
self.try_plan_native_operand(rhs, query_engine_state)
|
||||
.await?,
|
||||
) {
|
||||
(Some(left), Some(right)) => Some((left, right)),
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
// A declined binary uses the existing planner for its operands.
|
||||
let (mut left_input, mut left_context, mut right_input, mut right_context) = {
|
||||
let left = self.prom_expr_to_plan(lhs, query_engine_state).await?;
|
||||
let left_ctx = self.ctx.clone();
|
||||
let right = self.prom_expr_to_plan(rhs, query_engine_state).await?;
|
||||
let right_ctx = self.ctx.clone();
|
||||
(left, left_ctx, right, right_ctx)
|
||||
};
|
||||
let used_native = native.is_some();
|
||||
let (mut left_input, mut left_context, mut right_input, mut right_context) =
|
||||
if let Some(((left, left_ctx), (right, right_ctx))) = native {
|
||||
(left, left_ctx, right, right_ctx)
|
||||
} else {
|
||||
let left = self.prom_expr_to_plan(lhs, query_engine_state).await?;
|
||||
let left_ctx = self.ctx.clone();
|
||||
let right = self.prom_expr_to_plan(rhs, query_engine_state).await?;
|
||||
let right_ctx = self.ctx.clone();
|
||||
(left, left_ctx, right, right_ctx)
|
||||
};
|
||||
let left_field_columns = left_context.field_columns.clone();
|
||||
let left_time_index_column = left_context.time_index_column.clone();
|
||||
self.ctx = left_context.clone();
|
||||
@@ -1419,21 +1435,13 @@ impl PromPlanner {
|
||||
// Both operands are planned first because only the planned contexts tell a tag
|
||||
// from a value field. The rewrite merely adds matchers to a selector, so the
|
||||
// table reference, time index and field columns captured above stay valid.
|
||||
// Matcher propagation belongs to the legacy route: copying a matcher into the
|
||||
// other operand's scan drops the groups that have no partner, which is exactly
|
||||
// what lets the legacy join answer where Prometheus would report a duplicated
|
||||
// match group. A native operand must not be rewritten this way - the native
|
||||
// matcher checks the complete one-side groups itself, and filtering them first
|
||||
// would hide that error. So the rewrite only runs when neither operand is native.
|
||||
if !used_native
|
||||
&& let Some(rewritten) = matching_filters::propagate(
|
||||
binary_expr,
|
||||
&left_context.tag_columns,
|
||||
&right_context.tag_columns,
|
||||
&left_context.aggregation_field_labels,
|
||||
&right_context.aggregation_field_labels,
|
||||
)
|
||||
{
|
||||
if let Some(rewritten) = matching_filters::propagate(
|
||||
binary_expr,
|
||||
&left_context.tag_columns,
|
||||
&right_context.tag_columns,
|
||||
&left_context.aggregation_field_labels,
|
||||
&right_context.aggregation_field_labels,
|
||||
) {
|
||||
// A copied matcher belongs to the scan, not to the operand's identity:
|
||||
// `absent()` turns `selector_matcher` into the labels it reports, so the
|
||||
// re-planned context keeps the matchers the operand was written with.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+1348
-108
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -199,6 +199,7 @@ macro_rules! http_tests {
|
||||
test_remote_write_v2_histogram_float_prefix_native_route,
|
||||
test_remote_write_v2_histogram_native_annotations,
|
||||
test_remote_write_v2_histogram_native_arithmetic,
|
||||
test_remote_write_v2_histogram_native_composition,
|
||||
test_remote_write_v2_histogram_flush_native_consistency,
|
||||
test_remote_write_v2_histogram_native_partitions,
|
||||
test_remote_write_v2_histogram_native_range_function,
|
||||
@@ -5644,6 +5645,402 @@ pub async fn test_remote_write_v2_histogram_native_arithmetic(store_type: Storag
|
||||
);
|
||||
}
|
||||
|
||||
/// Executes a public PromQL request and checks the binary node actually used, without assuming how
|
||||
/// many fused producers or children the physical plan contains.
|
||||
async fn native_histogram_composition_query(
|
||||
client: &TestClient,
|
||||
observer: &NativeFusedObserver,
|
||||
route: &PromRouteObserver,
|
||||
query: &str,
|
||||
parameters: &str,
|
||||
) -> (StatusCode, Value) {
|
||||
observer.clear();
|
||||
let response = client
|
||||
.get(&format!(
|
||||
"/v1/prometheus/api/v1/{parameters}&query={}",
|
||||
encode(query)
|
||||
))
|
||||
.header("x-greptime-hints", "query.experimental_native_promql=true")
|
||||
.send()
|
||||
.await;
|
||||
let status = response.status();
|
||||
let body: Value = response.json().await;
|
||||
|
||||
let descriptions = observer
|
||||
.take_all(query)
|
||||
.iter()
|
||||
.map(|node| {
|
||||
datafusion::physical_plan::displayable(node.as_ref())
|
||||
.one_line()
|
||||
.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
descriptions.iter().any(|description| {
|
||||
description.lines().any(|line| {
|
||||
let mut words = line.split_whitespace();
|
||||
matches!(words.next(), Some("root" | "input" | "left" | "right"))
|
||||
&& words.next() == Some("Binary")
|
||||
})
|
||||
}),
|
||||
"`{query}` did not execute a native Binary: {descriptions:?}; response: {body}"
|
||||
);
|
||||
let nodes = plan_nodes(&route.take_plan());
|
||||
assert!(
|
||||
nodes
|
||||
.iter()
|
||||
.all(|node| { !node["name"].as_str().unwrap_or_default().contains("Join") }),
|
||||
"`{query}` executed a legacy join: {nodes:?}"
|
||||
);
|
||||
(status, body)
|
||||
}
|
||||
|
||||
/// Composed histogram frames retain their values, annotations and matching semantics over HTTP.
|
||||
pub async fn test_remote_write_v2_histogram_native_composition(store_type: StorageType) {
|
||||
let observer = Arc::new(NativeFusedObserver::default());
|
||||
let route = Arc::new(PromRouteObserver::default());
|
||||
let plugins = Plugins::default();
|
||||
let wrapper: PhysicalPlanWrapperRef = observer.clone();
|
||||
plugins.insert(wrapper);
|
||||
plugins.insert::<PromQueryInterceptorRef<servers::error::Error>>(route.clone());
|
||||
let instance = tests_integration::standalone::GreptimeDbStandaloneBuilder::new(
|
||||
"remote_write_v2_histogram_native_composition",
|
||||
)
|
||||
.with_default_store_type(store_type)
|
||||
.with_query_options(QueryOptions {
|
||||
experimental_native_promql: true,
|
||||
..Default::default()
|
||||
})
|
||||
.with_plugin(plugins)
|
||||
.build()
|
||||
.await;
|
||||
let server = build_test_prom_server(instance.fe_instance().clone(), false, None)
|
||||
.with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap())
|
||||
.build();
|
||||
let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
|
||||
|
||||
for (metric, job, instance_label, timestamp, count, sum) in [
|
||||
("native_comp_h", "api", "a", 1_000, 10, 20.0),
|
||||
// A later count of 1 makes losing the offset observable at the second range step.
|
||||
("native_comp_h", "api", "a", 121_000, 1, 2.0),
|
||||
// For this other job, losing @ end() instead makes the first step return 3 rather than 4.
|
||||
("native_comp_h", "anchor", "a", 1_000, 1, 2.0),
|
||||
("native_comp_h", "anchor", "a", 61_000, 10, 20.0),
|
||||
("native_comp_h", "anchor", "a", 121_000, 1, 2.0),
|
||||
("native_comp_h2", "api", "a", 1_000, 4, 8.0),
|
||||
("native_comp_h2", "other", "a", 1_000, 4, 8.0),
|
||||
("native_comp_dups", "duplicate", "a", 1_000, 4, 8.0),
|
||||
("native_comp_dups", "duplicate", "b", 1_000, 4, 8.0),
|
||||
] {
|
||||
write_native_histogram(
|
||||
&client,
|
||||
&[
|
||||
(prom_store::METRIC_NAME_LABEL, metric),
|
||||
("job", job),
|
||||
("instance", instance_label),
|
||||
],
|
||||
timestamp,
|
||||
count,
|
||||
sum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
for sql in [
|
||||
"CREATE TABLE native_comp_f (ts TIMESTAMP TIME INDEX, val DOUBLE, job STRING, PRIMARY KEY (job))",
|
||||
"INSERT INTO native_comp_f(job, val, ts) VALUES ('api', 2.0, 1000), ('duplicate', 2.0, 1000), ('api', 2.0, 61000), ('api', 9.0, 121000), ('anchor', 2.0, 61000), ('anchor', 9.0, 121000)",
|
||||
] {
|
||||
tests_integration::test_util::execute_sql(instance.fe_instance(), sql).await;
|
||||
}
|
||||
|
||||
// The first sample has two equal buckets of 5; h2 has two equal buckets of 2.
|
||||
// Repeat the same public requests after flushing so the source can be memtable or SST.
|
||||
for phase in ["memtable", "flushed"] {
|
||||
if phase == "flushed" {
|
||||
for table in [
|
||||
"native_comp_h",
|
||||
"native_comp_h2",
|
||||
"native_comp_dups",
|
||||
"native_comp_f",
|
||||
] {
|
||||
tests_integration::test_util::execute_sql(
|
||||
instance.fe_instance(),
|
||||
&format!("ADMIN FLUSH_TABLE('{table}')"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
for (query, count, sum, bucket) in [
|
||||
(
|
||||
"native_comp_f{job=\"api\"} * on(job) native_comp_h",
|
||||
"20",
|
||||
"40",
|
||||
"10",
|
||||
),
|
||||
(
|
||||
"native_comp_h / on(job) native_comp_f{job=\"api\"}",
|
||||
"5",
|
||||
"10",
|
||||
"2.5",
|
||||
),
|
||||
(
|
||||
"native_comp_h + on(job) native_comp_h2{job=\"api\"}",
|
||||
"14",
|
||||
"28",
|
||||
"7",
|
||||
),
|
||||
(
|
||||
"native_comp_h - on(job) native_comp_h2{job=\"api\"}",
|
||||
"6",
|
||||
"12",
|
||||
"3",
|
||||
),
|
||||
] {
|
||||
let (status, body) = native_histogram_composition_query(
|
||||
&client,
|
||||
&observer,
|
||||
&route,
|
||||
query,
|
||||
"query?time=1",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{phase}: {query}: {body}");
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({
|
||||
"status": "success",
|
||||
"data": {"resultType": "vector", "result": [{
|
||||
"metric": {"job": "api"},
|
||||
"histogram": [1.0, {"count": count, "sum": sum, "buckets": [
|
||||
[0, "0.7071067811865476", "1", bucket],
|
||||
[0, "1", "1.4142135623730951", bucket]
|
||||
]}]
|
||||
}]}
|
||||
}),
|
||||
"{phase}: {query}"
|
||||
);
|
||||
}
|
||||
|
||||
// These selectors read one unpartitioned histogram table but hold disjoint match groups.
|
||||
// The real histogram/histogram pair cannot emit for `*` or `/`, so only a synthetic float
|
||||
// can produce a result. Whether that result carries the unsupported-pair Info follows the
|
||||
// operation the synthesized sample enters, not whether the pair was synthesized:
|
||||
// Prometheus runs the same `doBinOp` on a filled sample, so a pairing it supports
|
||||
// (`float * histogram`, `histogram / float`) stays silent, while one it does not
|
||||
// (`float / histogram`) is dropped and reported exactly as a matched pair would be.
|
||||
let left = "native_comp_h{job=\"api\"}";
|
||||
let right = "native_comp_h{job=\"anchor\"}";
|
||||
for (operator, expected, dropped_op) in [
|
||||
(
|
||||
"* on(job) fill(1)",
|
||||
vec![("anchor", "1", "2", "1"), ("api", "10", "20", "5")],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"* on(job) fill_left(1)",
|
||||
vec![("anchor", "1", "2", "1")],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"* on(job) fill_right(1)",
|
||||
vec![("api", "10", "20", "5")],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/ on(job) fill_right(2)",
|
||||
vec![("api", "5", "10", "2.5")],
|
||||
None,
|
||||
),
|
||||
// `fill_left(2)` answers the anchor series with the synthesized float on the left, so
|
||||
// the pair is `float / histogram`: division by a histogram has no kernel, and the
|
||||
// sample Prometheus would drop here is the one this reports.
|
||||
("/ on(job) fill_left(2)", vec![], Some("/")),
|
||||
] {
|
||||
let query = format!("{left} {operator} {right}");
|
||||
let (status, body) = native_histogram_composition_query(
|
||||
&client,
|
||||
&observer,
|
||||
&route,
|
||||
&query,
|
||||
"query?time=1",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{phase}: {query}: {body}");
|
||||
assert_eq!(body["status"], "success", "{phase}: {query}: {body}");
|
||||
assert_eq!(
|
||||
body["data"]["resultType"], "vector",
|
||||
"{phase}: {query}: {body}"
|
||||
);
|
||||
match dropped_op {
|
||||
Some(op) => assert_eq!(
|
||||
body["infos"],
|
||||
json!([format!(
|
||||
"{op}: dropped native histogram samples because this binary operation is not supported for native histograms"
|
||||
)]),
|
||||
"{phase}: {query}: {body}"
|
||||
),
|
||||
None => assert!(
|
||||
body.get("infos").is_none(),
|
||||
"a supported fill is not a dropped pair: {phase}: {query}: {body}"
|
||||
),
|
||||
}
|
||||
assert!(body.get("warnings").is_none(), "{phase}: {query}: {body}");
|
||||
let mut rows = body["data"]["result"].as_array().unwrap().clone();
|
||||
rows.sort_by(|a, b| {
|
||||
a["metric"]["job"]
|
||||
.as_str()
|
||||
.cmp(&b["metric"]["job"].as_str())
|
||||
});
|
||||
let expected_rows: Vec<_> = expected
|
||||
.into_iter()
|
||||
.map(|(job, count, sum, bucket)| {
|
||||
// The anchor fixture's count=1 has bucket counts [0, 1]; the HTTP histogram
|
||||
// omits the first, zero-count bucket. The api fixture has two equal buckets.
|
||||
let buckets = if job == "anchor" {
|
||||
json!([[0, "1", "1.4142135623730951", bucket]])
|
||||
} else {
|
||||
json!([
|
||||
[0, "0.7071067811865476", "1", bucket],
|
||||
[0, "1", "1.4142135623730951", bucket]
|
||||
])
|
||||
};
|
||||
json!({
|
||||
"metric": {"job": job},
|
||||
"histogram": [1.0, {"count": count, "sum": sum, "buckets": buckets}]
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(rows, expected_rows, "{phase}: {query}: {body}");
|
||||
}
|
||||
// A present, matched histogram/histogram pair still drops and reports its existing Info.
|
||||
let query = format!("{left} * on(job) {left}");
|
||||
let (status, body) =
|
||||
native_histogram_composition_query(&client, &observer, &route, &query, "query?time=1")
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{phase}: {query}: {body}");
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({
|
||||
"status": "success",
|
||||
"data": {"resultType": "vector", "result": []},
|
||||
"infos": ["*: dropped native histogram samples because this binary operation is not supported for native histograms"]
|
||||
}),
|
||||
"{phase}: {query}"
|
||||
);
|
||||
|
||||
// `clamp` is native-only here, so legacy fallback cannot produce these answers.
|
||||
for (query, op) in [
|
||||
(
|
||||
"clamp(native_comp_f{job=\"api\"}, 1, 2) + on(job) native_comp_h",
|
||||
"+",
|
||||
),
|
||||
(
|
||||
"native_comp_h + on(job) clamp(native_comp_f{job=\"api\"}, 1, 2)",
|
||||
"+",
|
||||
),
|
||||
("native_comp_h * on(job) native_comp_h2{job=\"api\"}", "*"),
|
||||
] {
|
||||
let (status, body) = native_histogram_composition_query(
|
||||
&client,
|
||||
&observer,
|
||||
&route,
|
||||
query,
|
||||
"query?time=1",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{phase}: {query}: {body}");
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({
|
||||
"status": "success",
|
||||
"data": {"resultType": "vector", "result": []},
|
||||
"infos": [format!("{op}: dropped native histogram samples because this binary operation is not supported for native histograms")]
|
||||
}),
|
||||
"{phase}: {query}"
|
||||
);
|
||||
}
|
||||
|
||||
// A different match group, or a selector with no samples, must not fabricate a drop Info.
|
||||
for query in [
|
||||
"clamp(native_comp_f{job=\"api\"}, 1, 2) + on(job) native_comp_h2{job=\"other\"}",
|
||||
"native_comp_h * on(job) native_comp_h2{job=\"other\"}",
|
||||
"clamp(native_comp_f{job=\"absent\"}, 1, 2) + on(job) native_comp_h",
|
||||
"native_comp_h + on(job) clamp(native_comp_f{job=\"absent\"}, 1, 2)",
|
||||
] {
|
||||
let (status, body) = native_histogram_composition_query(
|
||||
&client,
|
||||
&observer,
|
||||
&route,
|
||||
query,
|
||||
"query?time=1",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{phase}: {query}: {body}");
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({"status": "success", "data": {"resultType": "vector", "result": []}}),
|
||||
"{phase}: {query}"
|
||||
);
|
||||
}
|
||||
|
||||
// The duplicate group is on the right. It is an error both when it matches and when it
|
||||
// does not: both sides have samples, so the native matcher's presence gate is open.
|
||||
for job in ["duplicate", "api"] {
|
||||
let query =
|
||||
format!("clamp(native_comp_f{{job=\"{job}\"}}, 1, 2) + on(job) native_comp_dups");
|
||||
let (status, body) = native_histogram_composition_query(
|
||||
&client,
|
||||
&observer,
|
||||
&route,
|
||||
&query,
|
||||
"query?time=1",
|
||||
)
|
||||
.await;
|
||||
assert!(!status.is_success(), "{phase}: {query}: {body}");
|
||||
assert_eq!(body["status"], "error", "{phase}: {query}: {body}");
|
||||
let error = body["error"].as_str().expect("matching error message");
|
||||
assert!(
|
||||
error.contains("found duplicate series for the match group {job=\"duplicate\"} on the right hand-side of the operation at 1000 ms; many-to-many matching not allowed"),
|
||||
"{phase}: {query}: {body}"
|
||||
);
|
||||
}
|
||||
|
||||
// The 121s end minus 1m reads the 61s histogram; @ start() reads the 61s float.
|
||||
// Both are 2 after clamp, at both points. A 121s float of 9 exposes a lost @ start().
|
||||
// The `anchor` job also exposes a lost @ end() at the first point; both jobs expose a
|
||||
// lost offset at the second point. Neither expected series carries its metric name.
|
||||
let query = "ceil(clamp(histogram_count(native_comp_h @ end() offset 1m), 1, 2) + on(job) native_comp_f{job=~\"api|anchor\"} @ start())";
|
||||
let (status, body) = native_histogram_composition_query(
|
||||
&client,
|
||||
&observer,
|
||||
&route,
|
||||
query,
|
||||
"query_range?start=61&end=121&step=60s",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{phase}: {query}: {body}");
|
||||
assert_eq!(body["status"], "success", "{phase}: {query}: {body}");
|
||||
assert_eq!(
|
||||
body["data"]["resultType"], "matrix",
|
||||
"{phase}: {query}: {body}"
|
||||
);
|
||||
let results = body["data"]["result"].as_array().expect("range result");
|
||||
assert_eq!(results.len(), 2, "{phase}: {query}: {body}");
|
||||
for job in ["api", "anchor"] {
|
||||
let expected = json!({
|
||||
"metric": {"job": job}, "values": [[61.0, "4.0"], [121.0, "4.0"]]
|
||||
});
|
||||
assert_eq!(
|
||||
results.iter().filter(|result| **result == expected).count(),
|
||||
1,
|
||||
"{phase}: {query}: {body}"
|
||||
);
|
||||
}
|
||||
assert!(body.get("infos").is_none(), "{phase}: {query}: {body}");
|
||||
assert!(body.get("warnings").is_none(), "{phase}: {query}: {body}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A native histogram query that has something to warn about puts it in the response.
|
||||
///
|
||||
/// `rate` over a gauge histogram is the case: the kernel drops nothing but reports that the input
|
||||
|
||||
Reference in New Issue
Block a user