refactor(promql): drop native runtime state at terminal (#10242)

Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
2026-11-09 13:58:56 +08:00
committed by GitHub
parent 9d5ef59c2b
commit ae9fb131e2
2 changed files with 748 additions and 556 deletions
File diff suppressed because it is too large Load Diff
+534 -94
View File
@@ -15,10 +15,11 @@
//! What the runtime has to get right: each layer's grid, per-node accounting, rejection and
//! release.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Weak};
use datafusion::error::DataFusionError;
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool};
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool, MemoryReservation};
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue};
use datatypes::arrow::array::{Array, Float64Array, UInt32Array};
use datatypes::arrow::buffer::{Buffer, ScalarBuffer};
@@ -68,6 +69,57 @@ use crate::vector_match::{
visited_pairs,
};
/// Watches the final release of a reservation without keeping its histogram payload alive.
#[derive(Debug)]
struct DropOrderPool {
inner: GreedyMemoryPool,
watch: std::sync::Mutex<Option<(String, Weak<HistogramBatch>)>>,
released: AtomicBool,
retained_on_release: AtomicBool,
}
impl std::fmt::Display for DropOrderPool {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "drop-order test pool")
}
}
impl MemoryPool for DropOrderPool {
fn name(&self) -> &str {
"drop-order test pool"
}
fn grow(&self, reservation: &MemoryReservation, additional: usize) {
self.inner.grow(reservation, additional);
}
fn try_grow(
&self,
reservation: &MemoryReservation,
additional: usize,
) -> Result<(), DataFusionError> {
self.inner.try_grow(reservation, additional)
}
fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
if reservation.size() == 0 {
let watch = self.watch.lock().unwrap();
if let Some((name, payload)) = watch.as_ref()
&& reservation.consumer().name() == name
{
self.released.store(true, Ordering::SeqCst);
self.retained_on_release
.store(payload.upgrade().is_some(), Ordering::SeqCst);
}
}
self.inner.shrink(reservation, shrink);
}
fn reserved(&self) -> usize {
self.inner.reserved()
}
}
fn grid(start: i64, step: i64, len: u64) -> Arc<EvalGrid> {
Arc::new(EvalGrid::try_new(GridSpec::try_new(start, step, len).unwrap()).unwrap())
}
@@ -360,11 +412,7 @@ fn window_and_limitk_collect_only_unified_contributions() {
group: Grouping::By(Arc::from([])),
},
));
let RuntimeOp::Vector {
vector: Some(vector),
..
} = raw_limitk.op
else {
let RuntimeOp::Vector { vector, .. } = raw_limitk.op else {
panic!("expected vector runtime node");
};
assert!(
@@ -378,11 +426,7 @@ fn window_and_limitk_collect_only_unified_contributions() {
};
let fragmented = map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]);
let fragmented_limitk = build(vector_op(fragmented, limitk()));
let RuntimeOp::Vector {
vector: Some(vector),
..
} = fragmented_limitk.op
else {
let RuntimeOp::Vector { vector, .. } = fragmented_limitk.op else {
panic!("expected vector runtime node");
};
assert!(
@@ -392,11 +436,7 @@ fn window_and_limitk_collect_only_unified_contributions() {
let unified = merge(map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]));
let unified_limitk = build(vector_op(unified, limitk()));
let RuntimeOp::Vector {
vector: Some(vector),
..
} = unified_limitk.op
else {
let RuntimeOp::Vector { vector, .. } = unified_limitk.op else {
panic!("expected vector runtime node");
};
assert!(
@@ -1439,13 +1479,15 @@ fn node_metrics(execution: &NativeExecutor) -> Vec<Weak<NodeMetrics>> {
input, parameter, ..
} => {
walk(input, out);
walk(parameter, out);
if let Some(parameter) = parameter {
walk(parameter, out);
}
}
RuntimeOp::ParameterizedHistogram {
input, parameters, ..
} => {
walk(input, out);
for parameter in parameters {
for parameter in parameters.iter().flatten() {
walk(parameter, out);
}
}
@@ -1460,7 +1502,7 @@ fn node_metrics(execution: &NativeExecutor) -> Vec<Weak<NodeMetrics>> {
}
}
let mut out = Vec::new();
walk(&execution.state().root, &mut out);
walk(&execution.state().live.as_ref().unwrap().root, &mut out);
out
}
@@ -1491,7 +1533,11 @@ fn gauge(set: &ExecutionPlanMetricsSet, name: &str) -> usize {
/// it: the trees these tests build have one node of that kind, and it is the root. The state owns
/// that tree, so the read is a borrow of it and the execution can be polled again afterwards.
fn root_holds(execution: &NativeExecutor, holds: fn(&RuntimeOp) -> bool) -> bool {
holds(&execution.state().root.op)
execution
.state()
.live
.as_ref()
.is_some_and(|live| holds(&live.root.op))
}
/// The payload a window root holds: its range evaluator.
@@ -1507,24 +1553,12 @@ fn holds_window(op: &RuntimeOp) -> bool {
/// The payload an instant root holds: the chunks of the series it is gathering.
fn holds_instant(op: &RuntimeOp) -> bool {
matches!(
op,
RuntimeOp::Instant {
instant: Some(_),
..
}
)
matches!(op, RuntimeOp::Instant { .. })
}
/// The payload a binary root holds: the operands it retained.
fn holds_binary(op: &RuntimeOp) -> bool {
matches!(
op,
RuntimeOp::Binary {
binary: Some(_),
..
}
)
matches!(op, RuntimeOp::Binary { .. })
}
/// Whether the root is the leaf that reads the first routing slot, which is what a tree of one
@@ -1533,6 +1567,154 @@ fn holds_input(op: &RuntimeOp) -> bool {
matches!(op, RuntimeOp::Input(id) if id.index() == 0)
}
/// Both queued input and retained operation payload must die before their reservations return.
#[test]
fn terminal_teardown_drops_payloads_before_returning_reservations() {
for retained in [false, true] {
let watched = Arc::new(DropOrderPool {
inner: GreedyMemoryPool::new(1 << 20),
watch: std::sync::Mutex::new(None),
released: AtomicBool::new(false),
retained_on_release: AtomicBool::new(false),
});
let pool: Arc<dyn MemoryPool> = watched.clone();
let grid = grid(0, 1, 1);
let schema = evaluated_schema(Arc::clone(&grid));
let root = if retained {
binary_of(
evaluated_input(Arc::clone(&grid)),
evaluated_input(Arc::clone(&grid)),
binary_matching(BinaryOp::Add),
)
} else {
evaluated_input(Arc::clone(&grid))
};
let tree = NativeTree::try_new(root).unwrap();
let mut execution = executor(&tree, Arc::clone(&pool));
let metrics = metrics_of(&execution);
let table = histogram_table(1, 2);
let weak = Arc::downgrade(&table);
let chunk = evaluated_histogram_chunk_of(&schema, metric(), 0, vec![Some(0)], &table, true);
drop(table);
execution.push_evaluated(InputId::new(0), chunk).unwrap();
if retained {
assert!(execution.next_chunk().unwrap().is_none());
}
assert!(weak.upgrade().is_some());
assert!(pool.reserved() > 0);
let consumer = if retained {
"native.binary at root"
} else {
"native.input at root (source_index=0)"
};
*watched.watch.lock().unwrap() = Some((consumer.to_string(), weak.clone()));
// Invalid routing fails while the queue or operation payload is still held.
let invalid = if retained { 2 } else { 1 };
let error = execution
.push_evaluated(
InputId::new(invalid),
evaluated_chunk(&schema, 0, vec![1.0], true),
)
.unwrap_err();
assert!(error.to_string().contains("no input"), "{error}");
assert!(watched.released.load(Ordering::SeqCst));
assert!(
!watched.retained_on_release.load(Ordering::SeqCst),
"{consumer}"
);
assert!(weak.upgrade().is_none());
assert!(execution.state().live.is_none());
assert_eq!(pool.reserved(), 0);
assert_eq!(gauge(&metrics, "native.root.current_reserved_bytes"), 0);
assert!(gauge(&metrics, "native.p0.root.peak_reserved_bytes") > 0);
assert!(
execution
.next_chunk()
.unwrap_err()
.to_string()
.contains("stopped after an earlier error")
);
}
}
/// A window finish failure is reported before its live execution and reservation are dropped.
#[test]
fn finishing_a_window_propagates_its_error_and_drops_the_live_tree() {
let tree = NativeTree::try_new(window(input(), grid(0, 60_000, 4))).unwrap();
let pool = pool();
let mut execution = executor(&tree, Arc::clone(&pool));
let metrics = metrics_of(&execution);
let nodes = node_metrics(&execution);
let schema = raw_schema();
execution
.push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false))
.unwrap();
drop(execution.next_chunk().unwrap());
assert!(pool.reserved() > 0);
// The source-level check normally catches truncation first; leave the evaluator open so the
// actual terminal finish pass exercises its own error and context propagation.
execution.state.live.as_mut().unwrap().inputs[0].open_source_series = false;
execution.end_of_input().unwrap();
let error = execution.next_chunk().unwrap_err();
assert!(
error.to_string().contains("native.window at root"),
"{error}"
);
assert!(
error.to_string().contains("ended before the last chunk"),
"{error}"
);
assert!(execution.state().live.is_none());
assert!(nodes.iter().all(|node| node.upgrade().is_none()));
assert_eq!(pool.reserved(), 0);
assert_eq!(gauge(&metrics, "native.root.current_reserved_bytes"), 0);
assert!(gauge(&metrics, "native.p0.root.peak_reserved_bytes") > 0);
assert!(
execution
.next_chunk()
.unwrap_err()
.to_string()
.contains("stopped after an earlier error")
);
}
/// The zero-input scalar operations retain their binding while live, not after completion.
#[test]
fn scalar_operations_drop_their_grid_and_topology_on_finish_or_failure() {
for vector in [false, true] {
for fails in [false, true] {
let grid = grid(0, 1, 2);
let weak = Arc::downgrade(&grid);
let time = eval_time(None, Arc::clone(&grid));
let tree = NativeTree::try_new(if vector { vector_of(time) } else { time }).unwrap();
let pool = if fails { bounded_pool(0) } else { pool() };
let mut execution = executor(&tree, Arc::clone(&pool));
let nodes = node_metrics(&execution);
drop(tree);
drop(grid);
execution.end_of_input().unwrap();
if fails {
assert!(execution.next_chunk().is_err());
} else {
drop(execution.next_chunk().unwrap().unwrap());
assert!(execution.next_chunk().unwrap().is_none());
}
assert!(weak.upgrade().is_none());
assert!(execution.state().live.is_none());
assert!(nodes.iter().all(|node| node.upgrade().is_none()));
assert_eq!(pool.reserved(), 0);
let error = execution.next_chunk().unwrap_err();
let expected = if fails {
"stopped after an earlier error"
} else {
"already finished"
};
assert!(error.to_string().contains(expected), "{error}");
}
}
}
/// A window over raw input evaluates onto its own output grid, and its values are the evaluator's.
///
/// The grid is compared by allocation identity, so a window that evaluated on a copy of its
@@ -2049,7 +2231,9 @@ fn a_binary_over_a_broadcast_registers_every_node_it_runs() {
// One direct pattern per level, on the nodes the constructor made; the borrow ends before the
// first push.
{
let RuntimeOp::Binary { left, right, .. } = &execution.state().root.op else {
let RuntimeOp::Binary { left, right, .. } =
&execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the binary node the tree built");
};
let RuntimeOp::Broadcast {
@@ -2302,7 +2486,7 @@ fn a_fanout_enqueue_failure_releases_every_recipient() {
let pool = bounded_pool(first_charge);
let mut execution = executor(&tree, Arc::clone(&pool));
let ids: Vec<_> = execution.state().input_ids(0).collect();
let names = &execution.state().inputs;
let names = &execution.state().live.as_ref().unwrap().inputs;
assert_ne!(
names[0].state.buffer.reservation.consumer().name(),
names[1].state.buffer.reservation.consumer().name()
@@ -3047,7 +3231,8 @@ fn a_window_that_merges_its_input_releases_what_it_collected() {
// The root is the window that collected the fragments, so its own reservation is what the peak
// below has to cover. The borrow ends before the close below.
let window_held = {
let RuntimeOp::Window { state, .. } = &execution.state().root.op else {
let RuntimeOp::Window { state, .. } = &execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the window the tree built");
};
state.buffer.reservation.size()
@@ -3120,7 +3305,9 @@ fn a_merge_flush_reserves_every_queued_output() {
// the node handed out under, and the evaluator that evaluated them. The borrow ends before the
// next pull.
let (queued, queued_bytes, retained, emitted, held) = {
let RuntimeOp::Window { window, state, .. } = &execution.state().root.op else {
let RuntimeOp::Window { window, state, .. } =
&execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the window the tree built");
};
let queued_bytes = state.queued.iter().map(|(_, bytes)| *bytes).sum::<usize>();
@@ -3209,7 +3396,7 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() {
window,
merge,
state,
} = &execution.state().root.op
} = &execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the window the tree built");
};
@@ -3241,7 +3428,7 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() {
merge,
state,
..
} = &execution.state().root.op
} = &execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the window the tree built");
};
@@ -5153,13 +5340,13 @@ fn a_source_charges_its_queue_and_gives_it_back() {
assert!(source.pop_source().unwrap().is_none());
}
/// Releasing a node drops its queue and returns the reservation once.
/// Dropping a node drops its queue and returns the reservation once.
///
/// A node stopped with chunks still queued holds nothing afterwards: the queue goes, and the bytes it
/// was reserved for go back to the pool rather than staying held for as long as the caller keeps the
/// stopped executor alive.
#[test]
fn releasing_a_node_drops_its_queue() {
fn dropping_a_node_drops_its_queue() {
let pool = pool();
let mut source = source_runtime(&pool);
let schema = raw_schema();
@@ -5175,14 +5362,12 @@ fn releasing_a_node_drops_its_queue() {
}
assert!(pool.reserved() > 0);
source.release();
assert!(
source.state.queued.is_empty(),
"the queue outlived its release"
);
assert_eq!(source.state.buffer.reservation.size(), 0);
let metrics = ExecutionPlanMetricsSet::new();
source.state.buffer.metrics.register("source", &metrics, 0);
drop(source);
assert_eq!(pool.reserved(), 0, "the reservation was not returned");
assert!(source.pop_source().unwrap().is_none());
assert_eq!(gauge(&metrics, "native.source.current_reserved_bytes"), 0);
assert!(gauge(&metrics, "native.p0.source.peak_reserved_bytes") > 0);
}
/// A chunk whose axis and fields disagree is refused, not read past the end of a field.
@@ -6333,14 +6518,14 @@ fn binary_runtime_indices_are_assigned_in_preorder() {
let RuntimeOp::Binary {
left,
binary: Some(root_binary),
binary: root_binary,
..
} = &execution.root.op
} = &execution.live.as_ref().unwrap().root.op
else {
panic!("the root is not a binary runtime");
};
let RuntimeOp::Binary {
binary: Some(inner_binary),
binary: inner_binary,
..
} = &left.op
else {
@@ -6396,9 +6581,9 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() {
let RuntimeOp::Binary {
left,
right,
binary: Some(root),
binary: root,
..
} = &execution.root.op
} = &execution.live.as_ref().unwrap().root.op
else {
panic!("expected root binary");
};
@@ -6419,11 +6604,7 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() {
panic!("expected histogram");
};
let binary_at = |node: &RuntimeNode| {
let RuntimeOp::Binary {
binary: Some(binary),
..
} = &node.op
else {
let RuntimeOp::Binary { binary, .. } = &node.op else {
panic!("expected vector binary");
};
assert!(Arc::ptr_eq(binary.evidence.as_ref().unwrap(), &evidence));
@@ -6438,10 +6619,16 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() {
assert!(Arc::ptr_eq(root.evidence.as_ref().unwrap(), &evidence));
assert_eq!(root.node_index, 0);
assert_eq!(binary_at(topk_input), 1);
assert_eq!(scalar_binary_at(topk_parameter), 2);
assert_eq!(scalar_binary_at(topk_parameter.as_ref().unwrap()), 2);
assert_eq!(binary_at(histogram_input), 3);
assert_eq!(scalar_binary_at(&histogram_parameters[0]), 4);
assert_eq!(scalar_binary_at(&histogram_parameters[1]), 5);
assert_eq!(
scalar_binary_at(histogram_parameters[0].as_ref().unwrap()),
4
);
assert_eq!(
scalar_binary_at(histogram_parameters[1].as_ref().unwrap()),
5
);
}
/// A source of evaluated chunks over `count` float fields.
@@ -8852,7 +9039,7 @@ fn queued_set_results_hold_their_labels_until_handed_on() {
.unwrap();
let start = |pool: &Arc<dyn MemoryPool>| {
let mut execution = executor(&tree, Arc::clone(pool));
match &execution.state().root.op {
match &execution.state().live.as_ref().unwrap().root.op {
RuntimeOp::Binary { state, .. } => assert_eq!(
state.buffer.reservation.consumer().name(),
"native.set at root"
@@ -10361,13 +10548,12 @@ fn an_aggregate_answers_nothing_until_its_whole_input_ends() {
// The root is the aggregate the tree built, and the fold is its own payload: what is read
// below is a copy of the flag, not a borrow held across the pulls.
let computed = |execution: &NativeExecutor| {
let RuntimeOp::Aggregate { aggregate, .. } = &execution.state().root.op else {
let RuntimeOp::Aggregate { aggregate, .. } =
&execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the aggregate the tree built");
};
aggregate
.as_ref()
.expect("the root holds an aggregate")
.computed
aggregate.computed
};
execution
@@ -10562,7 +10748,8 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() {
// reservation and the backing the queue grew to. The borrow ends inside the call, so the pull
// below is taken with nothing held.
let queue = |execution: &NativeExecutor| -> (usize, usize, usize, usize) {
let RuntimeOp::Aggregate { state, .. } = &execution.state().root.op else {
let RuntimeOp::Aggregate { state, .. } = &execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the aggregate the tree built");
};
(
@@ -10590,10 +10777,11 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() {
assert!(execution.next_chunk().unwrap().is_none());
// The terminal pull is the close: the queue's backing and the reservation go back with it, and
// the root reads released rather than only uncharged.
let (queued, _, held, capacity) = queue(&execution);
assert_eq!(queued, 0);
assert_eq!(capacity, 0, "the closed root kept the queue's backing");
assert_eq!(held, 0);
assert!(
execution.state().live.is_none(),
"the closed root was retained"
);
assert_eq!(execution.state().reserved_bytes(), 0);
assert_eq!(pool.reserved(), 0);
}
@@ -10642,15 +10830,16 @@ fn aggregate_fold_state_is_released_after_input() {
// The root is the aggregate the tree built: its fold holds one group and has not run. The
// borrow ends before the close below takes the state again.
{
let RuntimeOp::Aggregate { aggregate, .. } = &execution.state().root.op else {
let RuntimeOp::Aggregate { aggregate, .. } =
&execution.state().live.as_ref().unwrap().root.op
else {
panic!("the root is not the aggregate the tree built");
};
let aggregate = aggregate.as_ref().expect("the root holds an aggregate");
assert_eq!(aggregate.groups(), 1);
assert!(!aggregate.computed);
}
assert!(pool.reserved() > 0);
execution.state.close().unwrap();
execution.finish().unwrap();
assert_eq!(pool.reserved(), 0);
}
@@ -11136,6 +11325,265 @@ fn parameterized_topk_reserves_the_values_of_its_parameter() {
}
}
/// A consumed dynamic parameter drops charged scalar/broadcast state before the input is finished.
#[test]
fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() {
for histogram in [false, true] {
let grid = grid(1_000, 1_000, 2);
let input = evaluated_input(Arc::clone(&grid));
// Both sides are varying scalars: the broadcast retains one scalar series and emitted
// label history after producing the parameter.
let parameter = scalar_binary(
eval_time(None, Arc::clone(&grid)),
eval_time(None, Arc::clone(&grid)),
binary_matching(BinaryOp::Add),
);
let root = NativeNode {
output: input.output.clone(),
expr: if histogram {
NativeExpr::ParameterizedHistogram {
input: Box::new(input),
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)),
parameters: vec![parameter],
}
} else {
NativeExpr::ParameterizedTopK {
input: Box::new(input),
op: BoundVectorOp::TopK {
k: 0.0,
top: true,
group: Grouping::By(Arc::from([])),
},
parameter: Box::new(parameter),
}
},
};
let tree = NativeTree::try_new(root).unwrap();
let pool = pool();
let mut execution = executor(&tree, Arc::clone(&pool));
let metrics = metrics_of(&execution);
let parameter_name = if histogram {
"root.parameter0"
} else {
"root.parameter"
};
assert!(execution.next_chunk().unwrap().is_none());
let root = &execution.state().live.as_ref().unwrap().root;
match &root.op {
RuntimeOp::ParameterizedTopK {
parameter,
vector,
state,
..
} => {
assert!(parameter.is_none());
assert_eq!(
vector.parameter.as_deref(),
Some([Some(2.0), Some(4.0)].as_slice())
);
assert_eq!(
state.buffer.reservation.size(),
2 * std::mem::size_of::<Option<f64>>()
);
}
RuntimeOp::ParameterizedHistogram {
parameters,
values,
state,
..
} => {
assert!(parameters[0].is_none());
assert_eq!(
values[0].as_deref(),
Some([Some(2.0), Some(4.0)].as_slice())
);
assert_eq!(
state.buffer.reservation.size(),
2 * std::mem::size_of::<Option<f64>>()
);
}
_ => panic!("expected dynamic operator"),
}
assert_eq!(execution.state().reserved_bytes(), pool.reserved());
assert_eq!(pool.reserved(), 2 * std::mem::size_of::<Option<f64>>());
assert_eq!(
gauge(
&metrics,
&format!("native.{parameter_name}.current_reserved_bytes")
),
0
);
assert!(
gauge(
&metrics,
&format!("native.p0.{parameter_name}.peak_reserved_bytes")
) > 0
);
assert_eq!(
count(&metrics, parameter_name, NodeMetric::OUTPUT_BATCHES),
1
);
assert!(execution.finish().unwrap().is_empty());
assert_eq!(pool.reserved(), 0);
}
}
/// `scalar()` releases its own retained input after producing, but a map below it still holds
/// charged emitted-label history. Consuming the parameter must drop that subtree as well.
#[test]
fn dynamic_parameters_drop_scalar_of_map_history_before_terminal_close() {
for histogram in [false, true] {
let grid = grid(0, 1_000, 2);
let schema = evaluated_schema(Arc::clone(&grid));
let input = evaluated_input(Arc::clone(&grid));
let parameter = scalar_of(map(
evaluated_input_with_source_index(Arc::clone(&grid), 1),
vec![MathOp::Abs],
));
let root = NativeNode {
output: input.output.clone(),
expr: if histogram {
NativeExpr::ParameterizedHistogram {
input: Box::new(input),
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)),
parameters: vec![parameter],
}
} else {
NativeExpr::ParameterizedTopK {
input: Box::new(input),
op: BoundVectorOp::TopK {
k: 0.0,
top: true,
group: Grouping::By(Arc::from([])),
},
parameter: Box::new(parameter),
}
},
};
let tree = NativeTree::try_new(root).unwrap();
let pool = pool();
let mut execution = executor(&tree, Arc::clone(&pool));
let metrics = metrics_of(&execution);
let parameter_name = if histogram {
"root.parameter0"
} else {
"root.parameter"
};
let map_name = format!("{parameter_name}.input");
execution
.push_evaluated(
InputId::new(1),
evaluated_chunk(&schema, 0, vec![1.0, 1.0], true),
)
.unwrap();
assert!(execution.next_chunk().unwrap().is_none());
let map_current = format!("native.{map_name}.current_reserved_bytes");
assert!(
gauge(&metrics, &map_current) > 0,
"map did not retain emitted history"
);
// Drive exactly the finishing stage that produces the scalar, before the executor closes.
assert!(execution.live_mut().unwrap().pull(true).unwrap().is_none());
assert_eq!(gauge(&metrics, &map_current), 0);
assert!(
gauge(
&metrics,
&format!("native.p0.{map_name}.peak_reserved_bytes")
) > 0
);
assert_eq!(count(&metrics, &map_name, NodeMetric::OUTPUT_BATCHES), 1);
let root = &execution.state().live.as_ref().unwrap().root;
match &root.op {
RuntimeOp::ParameterizedTopK { parameter, .. } => assert!(parameter.is_none()),
RuntimeOp::ParameterizedHistogram { parameters, .. } => {
assert!(parameters[0].is_none())
}
_ => panic!("expected dynamic operator"),
}
assert_eq!(execution.state().reserved_bytes(), pool.reserved());
execution.live_mut().unwrap().root.finish_windows().unwrap();
drop(execution);
assert_eq!(pool.reserved(), 0);
}
}
/// A settled parameter with an open window must report the window's finish error before dropping it.
#[test]
fn dynamic_parameters_finish_windows_before_dropping_the_subtree() {
for histogram in [false, true] {
let grid = grid(0, 1_000, 2);
let input = evaluated_input(Arc::clone(&grid));
let parameter_window = window(raw_input_of(1), Arc::clone(&grid));
let parameter = scalar_of(parameter_window);
let root = NativeNode {
output: input.output.clone(),
expr: if histogram {
NativeExpr::ParameterizedHistogram {
input: Box::new(input),
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)),
parameters: vec![parameter],
}
} else {
NativeExpr::ParameterizedTopK {
input: Box::new(input),
op: BoundVectorOp::TopK {
k: 0.0,
top: true,
group: Grouping::By(Arc::from([])),
},
parameter: Box::new(parameter),
}
},
};
let tree = NativeTree::try_new(root).unwrap();
let pool = pool();
let mut execution = executor(&tree, Arc::clone(&pool));
let live = execution.state.live.as_mut().unwrap();
let parameter = match &mut live.root.op {
RuntimeOp::ParameterizedTopK { parameter, .. } => parameter.as_deref_mut().unwrap(),
RuntimeOp::ParameterizedHistogram { parameters, .. } => parameters[0].as_mut().unwrap(),
_ => panic!("expected dynamic operator"),
};
let RuntimeOp::ScalarOf {
input,
produced,
state,
..
} = &mut parameter.op
else {
panic!("expected scalar() parameter");
};
let RuntimeOp::Window { window, .. } = &mut input.op else {
panic!("expected parameter window");
};
// Force a queued scalar and an open evaluator, an otherwise unreachable invariant break,
// so the real finish failure cannot be bypassed by ordinary EOF checks.
let chunk = raw_chunk(&raw_schema(), &[0], &[1.0], false);
drop(window.as_mut().unwrap().push(&chunk).unwrap());
let bytes = grid_chunk_bytes(&grid).unwrap();
state.buffer.grow(bytes).unwrap();
state.queued.push_back((
NativeChunk::Evaluated(super::scalar_chunk(Arc::clone(&grid), 1.0).unwrap()),
bytes,
));
*produced = true;
execution.inputs_finished = true;
let error = execution.next_chunk().unwrap_err();
assert!(
error.to_string().contains("ended before the last chunk"),
"{error}"
);
assert!(
error
.to_string()
.contains("native.window at root.parameter"),
"{error}"
);
assert!(execution.state().live.is_none());
assert_eq!(pool.reserved(), 0);
}
}
/// A parameterized `topk` released before it computes gives back everything it holds: the input it
/// retained, and with it the samples and the label allocations those chunks keep alive.
///
@@ -11191,7 +11639,7 @@ fn a_parameterized_topk_released_before_it_computes_drops_its_retained_input() {
pool.reserved() > 0,
"the retained chunk was not charged for"
);
execution.state.close().unwrap();
drop(execution);
assert_eq!(pool.reserved(), 0, "the reservation was not returned");
assert_eq!(
Arc::strong_count(&job),
@@ -11311,8 +11759,7 @@ fn parameterized_histogram_hands_over_its_results_from_its_own_queue() {
("host=b".to_string(), vec![(0, Some(1.5)), (1, Some(1.5))]),
]
);
// The queue is drained, so what is left is the values the parameters were read into.
execution.state.close().unwrap();
// The terminal pull dropped the queue and parameter values with the live tree.
assert_eq!(pool.reserved(), 0);
}
@@ -11936,7 +12383,7 @@ fn scalar_of_reports_what_it_holds_while_its_input_is_retained() {
pool.reserved(),
"the reported reservation is not what the node holds"
);
execution.state.close().unwrap();
drop(execution);
assert_eq!(pool.reserved(), 0, "the reservation was not returned");
}
@@ -12559,11 +13006,9 @@ fn a_label_change_admits_the_label_it_writes_before_building_it() {
assert_eq!(pool.reserved(), 0);
}
/// An ended execution keeps its tree and its registered counters, but neither the input schema nor
/// the grid it is addressed on. The leaf itself stays -- the root still says which routing slot it
/// is, and the handles the caller registered before the run still read what the nodes recorded --
/// but the slot that fed it is taken. Both ways an execution ends are covered: the answer is drained,
/// and a rejected push stops it first.
/// An ended execution drops its tree, input schema and grid, while registered counters survive.
/// Both ways an execution ends are covered: the answer is drained, and a rejected push stops it
/// first.
#[test]
fn an_ended_execution_keeps_no_input_schema_or_grid() {
for fails in [false, true] {
@@ -12642,13 +13087,8 @@ fn an_ended_execution_keeps_no_input_schema_or_grid() {
assert_eq!(execution.state().reserved_bytes(), 0);
assert_eq!(pool.reserved(), 0);
assert!(
execution.state().inputs.is_empty(),
"the ended execution kept an input slot"
);
assert_eq!(
execution.state().inputs.capacity(),
0,
"the ended execution kept the slot vector's backing"
execution.state().live.is_none(),
"the ended execution kept its input slots"
);
let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES);
assert_eq!(
@@ -12662,8 +13102,8 @@ fn an_ended_execution_keeps_no_input_schema_or_grid() {
"the peak the leaf reached did not survive the end of the execution"
);
assert!(
root_holds(&execution, holds_input),
"the ended execution replaced the root it was built from"
!root_holds(&execution, holds_input),
"the ended execution kept its root"
);
}
}