mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 16:24:07 +00:00
refactor(promql): drop native runtime state at terminal (#10242)
Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
+214
-462
File diff suppressed because it is too large
Load Diff
@@ -15,10 +15,11 @@
|
||||
//! What the runtime has to get right: each layer's grid, per-node accounting, rejection and
|
||||
//! release.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use datafusion::error::DataFusionError;
|
||||
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool};
|
||||
use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool, MemoryReservation};
|
||||
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue};
|
||||
use datatypes::arrow::array::{Array, Float64Array, UInt32Array};
|
||||
use datatypes::arrow::buffer::{Buffer, ScalarBuffer};
|
||||
@@ -68,6 +69,57 @@ use crate::vector_match::{
|
||||
visited_pairs,
|
||||
};
|
||||
|
||||
/// Watches the final release of a reservation without keeping its histogram payload alive.
|
||||
#[derive(Debug)]
|
||||
struct DropOrderPool {
|
||||
inner: GreedyMemoryPool,
|
||||
watch: std::sync::Mutex<Option<(String, Weak<HistogramBatch>)>>,
|
||||
released: AtomicBool,
|
||||
retained_on_release: AtomicBool,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DropOrderPool {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "drop-order test pool")
|
||||
}
|
||||
}
|
||||
|
||||
impl MemoryPool for DropOrderPool {
|
||||
fn name(&self) -> &str {
|
||||
"drop-order test pool"
|
||||
}
|
||||
|
||||
fn grow(&self, reservation: &MemoryReservation, additional: usize) {
|
||||
self.inner.grow(reservation, additional);
|
||||
}
|
||||
|
||||
fn try_grow(
|
||||
&self,
|
||||
reservation: &MemoryReservation,
|
||||
additional: usize,
|
||||
) -> Result<(), DataFusionError> {
|
||||
self.inner.try_grow(reservation, additional)
|
||||
}
|
||||
|
||||
fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
|
||||
if reservation.size() == 0 {
|
||||
let watch = self.watch.lock().unwrap();
|
||||
if let Some((name, payload)) = watch.as_ref()
|
||||
&& reservation.consumer().name() == name
|
||||
{
|
||||
self.released.store(true, Ordering::SeqCst);
|
||||
self.retained_on_release
|
||||
.store(payload.upgrade().is_some(), Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
self.inner.shrink(reservation, shrink);
|
||||
}
|
||||
|
||||
fn reserved(&self) -> usize {
|
||||
self.inner.reserved()
|
||||
}
|
||||
}
|
||||
|
||||
fn grid(start: i64, step: i64, len: u64) -> Arc<EvalGrid> {
|
||||
Arc::new(EvalGrid::try_new(GridSpec::try_new(start, step, len).unwrap()).unwrap())
|
||||
}
|
||||
@@ -360,11 +412,7 @@ fn window_and_limitk_collect_only_unified_contributions() {
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
));
|
||||
let RuntimeOp::Vector {
|
||||
vector: Some(vector),
|
||||
..
|
||||
} = raw_limitk.op
|
||||
else {
|
||||
let RuntimeOp::Vector { vector, .. } = raw_limitk.op else {
|
||||
panic!("expected vector runtime node");
|
||||
};
|
||||
assert!(
|
||||
@@ -378,11 +426,7 @@ fn window_and_limitk_collect_only_unified_contributions() {
|
||||
};
|
||||
let fragmented = map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]);
|
||||
let fragmented_limitk = build(vector_op(fragmented, limitk()));
|
||||
let RuntimeOp::Vector {
|
||||
vector: Some(vector),
|
||||
..
|
||||
} = fragmented_limitk.op
|
||||
else {
|
||||
let RuntimeOp::Vector { vector, .. } = fragmented_limitk.op else {
|
||||
panic!("expected vector runtime node");
|
||||
};
|
||||
assert!(
|
||||
@@ -392,11 +436,7 @@ fn window_and_limitk_collect_only_unified_contributions() {
|
||||
|
||||
let unified = merge(map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]));
|
||||
let unified_limitk = build(vector_op(unified, limitk()));
|
||||
let RuntimeOp::Vector {
|
||||
vector: Some(vector),
|
||||
..
|
||||
} = unified_limitk.op
|
||||
else {
|
||||
let RuntimeOp::Vector { vector, .. } = unified_limitk.op else {
|
||||
panic!("expected vector runtime node");
|
||||
};
|
||||
assert!(
|
||||
@@ -1439,13 +1479,15 @@ fn node_metrics(execution: &NativeExecutor) -> Vec<Weak<NodeMetrics>> {
|
||||
input, parameter, ..
|
||||
} => {
|
||||
walk(input, out);
|
||||
walk(parameter, out);
|
||||
if let Some(parameter) = parameter {
|
||||
walk(parameter, out);
|
||||
}
|
||||
}
|
||||
RuntimeOp::ParameterizedHistogram {
|
||||
input, parameters, ..
|
||||
} => {
|
||||
walk(input, out);
|
||||
for parameter in parameters {
|
||||
for parameter in parameters.iter().flatten() {
|
||||
walk(parameter, out);
|
||||
}
|
||||
}
|
||||
@@ -1460,7 +1502,7 @@ fn node_metrics(execution: &NativeExecutor) -> Vec<Weak<NodeMetrics>> {
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
walk(&execution.state().root, &mut out);
|
||||
walk(&execution.state().live.as_ref().unwrap().root, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
@@ -1491,7 +1533,11 @@ fn gauge(set: &ExecutionPlanMetricsSet, name: &str) -> usize {
|
||||
/// it: the trees these tests build have one node of that kind, and it is the root. The state owns
|
||||
/// that tree, so the read is a borrow of it and the execution can be polled again afterwards.
|
||||
fn root_holds(execution: &NativeExecutor, holds: fn(&RuntimeOp) -> bool) -> bool {
|
||||
holds(&execution.state().root.op)
|
||||
execution
|
||||
.state()
|
||||
.live
|
||||
.as_ref()
|
||||
.is_some_and(|live| holds(&live.root.op))
|
||||
}
|
||||
|
||||
/// The payload a window root holds: its range evaluator.
|
||||
@@ -1507,24 +1553,12 @@ fn holds_window(op: &RuntimeOp) -> bool {
|
||||
|
||||
/// The payload an instant root holds: the chunks of the series it is gathering.
|
||||
fn holds_instant(op: &RuntimeOp) -> bool {
|
||||
matches!(
|
||||
op,
|
||||
RuntimeOp::Instant {
|
||||
instant: Some(_),
|
||||
..
|
||||
}
|
||||
)
|
||||
matches!(op, RuntimeOp::Instant { .. })
|
||||
}
|
||||
|
||||
/// The payload a binary root holds: the operands it retained.
|
||||
fn holds_binary(op: &RuntimeOp) -> bool {
|
||||
matches!(
|
||||
op,
|
||||
RuntimeOp::Binary {
|
||||
binary: Some(_),
|
||||
..
|
||||
}
|
||||
)
|
||||
matches!(op, RuntimeOp::Binary { .. })
|
||||
}
|
||||
|
||||
/// Whether the root is the leaf that reads the first routing slot, which is what a tree of one
|
||||
@@ -1533,6 +1567,154 @@ fn holds_input(op: &RuntimeOp) -> bool {
|
||||
matches!(op, RuntimeOp::Input(id) if id.index() == 0)
|
||||
}
|
||||
|
||||
/// Both queued input and retained operation payload must die before their reservations return.
|
||||
#[test]
|
||||
fn terminal_teardown_drops_payloads_before_returning_reservations() {
|
||||
for retained in [false, true] {
|
||||
let watched = Arc::new(DropOrderPool {
|
||||
inner: GreedyMemoryPool::new(1 << 20),
|
||||
watch: std::sync::Mutex::new(None),
|
||||
released: AtomicBool::new(false),
|
||||
retained_on_release: AtomicBool::new(false),
|
||||
});
|
||||
let pool: Arc<dyn MemoryPool> = watched.clone();
|
||||
let grid = grid(0, 1, 1);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let root = if retained {
|
||||
binary_of(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
binary_matching(BinaryOp::Add),
|
||||
)
|
||||
} else {
|
||||
evaluated_input(Arc::clone(&grid))
|
||||
};
|
||||
let tree = NativeTree::try_new(root).unwrap();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let metrics = metrics_of(&execution);
|
||||
let table = histogram_table(1, 2);
|
||||
let weak = Arc::downgrade(&table);
|
||||
let chunk = evaluated_histogram_chunk_of(&schema, metric(), 0, vec![Some(0)], &table, true);
|
||||
drop(table);
|
||||
execution.push_evaluated(InputId::new(0), chunk).unwrap();
|
||||
if retained {
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
}
|
||||
assert!(weak.upgrade().is_some());
|
||||
assert!(pool.reserved() > 0);
|
||||
let consumer = if retained {
|
||||
"native.binary at root"
|
||||
} else {
|
||||
"native.input at root (source_index=0)"
|
||||
};
|
||||
*watched.watch.lock().unwrap() = Some((consumer.to_string(), weak.clone()));
|
||||
|
||||
// Invalid routing fails while the queue or operation payload is still held.
|
||||
let invalid = if retained { 2 } else { 1 };
|
||||
let error = execution
|
||||
.push_evaluated(
|
||||
InputId::new(invalid),
|
||||
evaluated_chunk(&schema, 0, vec![1.0], true),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("no input"), "{error}");
|
||||
assert!(watched.released.load(Ordering::SeqCst));
|
||||
assert!(
|
||||
!watched.retained_on_release.load(Ordering::SeqCst),
|
||||
"{consumer}"
|
||||
);
|
||||
assert!(weak.upgrade().is_none());
|
||||
assert!(execution.state().live.is_none());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
assert_eq!(gauge(&metrics, "native.root.current_reserved_bytes"), 0);
|
||||
assert!(gauge(&metrics, "native.p0.root.peak_reserved_bytes") > 0);
|
||||
assert!(
|
||||
execution
|
||||
.next_chunk()
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("stopped after an earlier error")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A window finish failure is reported before its live execution and reservation are dropped.
|
||||
#[test]
|
||||
fn finishing_a_window_propagates_its_error_and_drops_the_live_tree() {
|
||||
let tree = NativeTree::try_new(window(input(), grid(0, 60_000, 4))).unwrap();
|
||||
let pool = pool();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let metrics = metrics_of(&execution);
|
||||
let nodes = node_metrics(&execution);
|
||||
let schema = raw_schema();
|
||||
execution
|
||||
.push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false))
|
||||
.unwrap();
|
||||
drop(execution.next_chunk().unwrap());
|
||||
assert!(pool.reserved() > 0);
|
||||
// The source-level check normally catches truncation first; leave the evaluator open so the
|
||||
// actual terminal finish pass exercises its own error and context propagation.
|
||||
execution.state.live.as_mut().unwrap().inputs[0].open_source_series = false;
|
||||
execution.end_of_input().unwrap();
|
||||
let error = execution.next_chunk().unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("native.window at root"),
|
||||
"{error}"
|
||||
);
|
||||
assert!(
|
||||
error.to_string().contains("ended before the last chunk"),
|
||||
"{error}"
|
||||
);
|
||||
assert!(execution.state().live.is_none());
|
||||
assert!(nodes.iter().all(|node| node.upgrade().is_none()));
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
assert_eq!(gauge(&metrics, "native.root.current_reserved_bytes"), 0);
|
||||
assert!(gauge(&metrics, "native.p0.root.peak_reserved_bytes") > 0);
|
||||
assert!(
|
||||
execution
|
||||
.next_chunk()
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("stopped after an earlier error")
|
||||
);
|
||||
}
|
||||
|
||||
/// The zero-input scalar operations retain their binding while live, not after completion.
|
||||
#[test]
|
||||
fn scalar_operations_drop_their_grid_and_topology_on_finish_or_failure() {
|
||||
for vector in [false, true] {
|
||||
for fails in [false, true] {
|
||||
let grid = grid(0, 1, 2);
|
||||
let weak = Arc::downgrade(&grid);
|
||||
let time = eval_time(None, Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(if vector { vector_of(time) } else { time }).unwrap();
|
||||
let pool = if fails { bounded_pool(0) } else { pool() };
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let nodes = node_metrics(&execution);
|
||||
drop(tree);
|
||||
drop(grid);
|
||||
execution.end_of_input().unwrap();
|
||||
if fails {
|
||||
assert!(execution.next_chunk().is_err());
|
||||
} else {
|
||||
drop(execution.next_chunk().unwrap().unwrap());
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
}
|
||||
assert!(weak.upgrade().is_none());
|
||||
assert!(execution.state().live.is_none());
|
||||
assert!(nodes.iter().all(|node| node.upgrade().is_none()));
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
let error = execution.next_chunk().unwrap_err();
|
||||
let expected = if fails {
|
||||
"stopped after an earlier error"
|
||||
} else {
|
||||
"already finished"
|
||||
};
|
||||
assert!(error.to_string().contains(expected), "{error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A window over raw input evaluates onto its own output grid, and its values are the evaluator's.
|
||||
///
|
||||
/// The grid is compared by allocation identity, so a window that evaluated on a copy of its
|
||||
@@ -2049,7 +2231,9 @@ fn a_binary_over_a_broadcast_registers_every_node_it_runs() {
|
||||
// One direct pattern per level, on the nodes the constructor made; the borrow ends before the
|
||||
// first push.
|
||||
{
|
||||
let RuntimeOp::Binary { left, right, .. } = &execution.state().root.op else {
|
||||
let RuntimeOp::Binary { left, right, .. } =
|
||||
&execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the binary node the tree built");
|
||||
};
|
||||
let RuntimeOp::Broadcast {
|
||||
@@ -2302,7 +2486,7 @@ fn a_fanout_enqueue_failure_releases_every_recipient() {
|
||||
let pool = bounded_pool(first_charge);
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let ids: Vec<_> = execution.state().input_ids(0).collect();
|
||||
let names = &execution.state().inputs;
|
||||
let names = &execution.state().live.as_ref().unwrap().inputs;
|
||||
assert_ne!(
|
||||
names[0].state.buffer.reservation.consumer().name(),
|
||||
names[1].state.buffer.reservation.consumer().name()
|
||||
@@ -3047,7 +3231,8 @@ fn a_window_that_merges_its_input_releases_what_it_collected() {
|
||||
// The root is the window that collected the fragments, so its own reservation is what the peak
|
||||
// below has to cover. The borrow ends before the close below.
|
||||
let window_held = {
|
||||
let RuntimeOp::Window { state, .. } = &execution.state().root.op else {
|
||||
let RuntimeOp::Window { state, .. } = &execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the window the tree built");
|
||||
};
|
||||
state.buffer.reservation.size()
|
||||
@@ -3120,7 +3305,9 @@ fn a_merge_flush_reserves_every_queued_output() {
|
||||
// the node handed out under, and the evaluator that evaluated them. The borrow ends before the
|
||||
// next pull.
|
||||
let (queued, queued_bytes, retained, emitted, held) = {
|
||||
let RuntimeOp::Window { window, state, .. } = &execution.state().root.op else {
|
||||
let RuntimeOp::Window { window, state, .. } =
|
||||
&execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the window the tree built");
|
||||
};
|
||||
let queued_bytes = state.queued.iter().map(|(_, bytes)| *bytes).sum::<usize>();
|
||||
@@ -3209,7 +3396,7 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() {
|
||||
window,
|
||||
merge,
|
||||
state,
|
||||
} = &execution.state().root.op
|
||||
} = &execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the window the tree built");
|
||||
};
|
||||
@@ -3241,7 +3428,7 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() {
|
||||
merge,
|
||||
state,
|
||||
..
|
||||
} = &execution.state().root.op
|
||||
} = &execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the window the tree built");
|
||||
};
|
||||
@@ -5153,13 +5340,13 @@ fn a_source_charges_its_queue_and_gives_it_back() {
|
||||
assert!(source.pop_source().unwrap().is_none());
|
||||
}
|
||||
|
||||
/// Releasing a node drops its queue and returns the reservation once.
|
||||
/// Dropping a node drops its queue and returns the reservation once.
|
||||
///
|
||||
/// A node stopped with chunks still queued holds nothing afterwards: the queue goes, and the bytes it
|
||||
/// was reserved for go back to the pool rather than staying held for as long as the caller keeps the
|
||||
/// stopped executor alive.
|
||||
#[test]
|
||||
fn releasing_a_node_drops_its_queue() {
|
||||
fn dropping_a_node_drops_its_queue() {
|
||||
let pool = pool();
|
||||
let mut source = source_runtime(&pool);
|
||||
let schema = raw_schema();
|
||||
@@ -5175,14 +5362,12 @@ fn releasing_a_node_drops_its_queue() {
|
||||
}
|
||||
assert!(pool.reserved() > 0);
|
||||
|
||||
source.release();
|
||||
assert!(
|
||||
source.state.queued.is_empty(),
|
||||
"the queue outlived its release"
|
||||
);
|
||||
assert_eq!(source.state.buffer.reservation.size(), 0);
|
||||
let metrics = ExecutionPlanMetricsSet::new();
|
||||
source.state.buffer.metrics.register("source", &metrics, 0);
|
||||
drop(source);
|
||||
assert_eq!(pool.reserved(), 0, "the reservation was not returned");
|
||||
assert!(source.pop_source().unwrap().is_none());
|
||||
assert_eq!(gauge(&metrics, "native.source.current_reserved_bytes"), 0);
|
||||
assert!(gauge(&metrics, "native.p0.source.peak_reserved_bytes") > 0);
|
||||
}
|
||||
|
||||
/// A chunk whose axis and fields disagree is refused, not read past the end of a field.
|
||||
@@ -6333,14 +6518,14 @@ fn binary_runtime_indices_are_assigned_in_preorder() {
|
||||
|
||||
let RuntimeOp::Binary {
|
||||
left,
|
||||
binary: Some(root_binary),
|
||||
binary: root_binary,
|
||||
..
|
||||
} = &execution.root.op
|
||||
} = &execution.live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not a binary runtime");
|
||||
};
|
||||
let RuntimeOp::Binary {
|
||||
binary: Some(inner_binary),
|
||||
binary: inner_binary,
|
||||
..
|
||||
} = &left.op
|
||||
else {
|
||||
@@ -6396,9 +6581,9 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() {
|
||||
let RuntimeOp::Binary {
|
||||
left,
|
||||
right,
|
||||
binary: Some(root),
|
||||
binary: root,
|
||||
..
|
||||
} = &execution.root.op
|
||||
} = &execution.live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("expected root binary");
|
||||
};
|
||||
@@ -6419,11 +6604,7 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() {
|
||||
panic!("expected histogram");
|
||||
};
|
||||
let binary_at = |node: &RuntimeNode| {
|
||||
let RuntimeOp::Binary {
|
||||
binary: Some(binary),
|
||||
..
|
||||
} = &node.op
|
||||
else {
|
||||
let RuntimeOp::Binary { binary, .. } = &node.op else {
|
||||
panic!("expected vector binary");
|
||||
};
|
||||
assert!(Arc::ptr_eq(binary.evidence.as_ref().unwrap(), &evidence));
|
||||
@@ -6438,10 +6619,16 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() {
|
||||
assert!(Arc::ptr_eq(root.evidence.as_ref().unwrap(), &evidence));
|
||||
assert_eq!(root.node_index, 0);
|
||||
assert_eq!(binary_at(topk_input), 1);
|
||||
assert_eq!(scalar_binary_at(topk_parameter), 2);
|
||||
assert_eq!(scalar_binary_at(topk_parameter.as_ref().unwrap()), 2);
|
||||
assert_eq!(binary_at(histogram_input), 3);
|
||||
assert_eq!(scalar_binary_at(&histogram_parameters[0]), 4);
|
||||
assert_eq!(scalar_binary_at(&histogram_parameters[1]), 5);
|
||||
assert_eq!(
|
||||
scalar_binary_at(histogram_parameters[0].as_ref().unwrap()),
|
||||
4
|
||||
);
|
||||
assert_eq!(
|
||||
scalar_binary_at(histogram_parameters[1].as_ref().unwrap()),
|
||||
5
|
||||
);
|
||||
}
|
||||
|
||||
/// A source of evaluated chunks over `count` float fields.
|
||||
@@ -8852,7 +9039,7 @@ fn queued_set_results_hold_their_labels_until_handed_on() {
|
||||
.unwrap();
|
||||
let start = |pool: &Arc<dyn MemoryPool>| {
|
||||
let mut execution = executor(&tree, Arc::clone(pool));
|
||||
match &execution.state().root.op {
|
||||
match &execution.state().live.as_ref().unwrap().root.op {
|
||||
RuntimeOp::Binary { state, .. } => assert_eq!(
|
||||
state.buffer.reservation.consumer().name(),
|
||||
"native.set at root"
|
||||
@@ -10361,13 +10548,12 @@ fn an_aggregate_answers_nothing_until_its_whole_input_ends() {
|
||||
// The root is the aggregate the tree built, and the fold is its own payload: what is read
|
||||
// below is a copy of the flag, not a borrow held across the pulls.
|
||||
let computed = |execution: &NativeExecutor| {
|
||||
let RuntimeOp::Aggregate { aggregate, .. } = &execution.state().root.op else {
|
||||
let RuntimeOp::Aggregate { aggregate, .. } =
|
||||
&execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the aggregate the tree built");
|
||||
};
|
||||
aggregate
|
||||
.as_ref()
|
||||
.expect("the root holds an aggregate")
|
||||
.computed
|
||||
aggregate.computed
|
||||
};
|
||||
|
||||
execution
|
||||
@@ -10562,7 +10748,8 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() {
|
||||
// reservation and the backing the queue grew to. The borrow ends inside the call, so the pull
|
||||
// below is taken with nothing held.
|
||||
let queue = |execution: &NativeExecutor| -> (usize, usize, usize, usize) {
|
||||
let RuntimeOp::Aggregate { state, .. } = &execution.state().root.op else {
|
||||
let RuntimeOp::Aggregate { state, .. } = &execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the aggregate the tree built");
|
||||
};
|
||||
(
|
||||
@@ -10590,10 +10777,11 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() {
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
// The terminal pull is the close: the queue's backing and the reservation go back with it, and
|
||||
// the root reads released rather than only uncharged.
|
||||
let (queued, _, held, capacity) = queue(&execution);
|
||||
assert_eq!(queued, 0);
|
||||
assert_eq!(capacity, 0, "the closed root kept the queue's backing");
|
||||
assert_eq!(held, 0);
|
||||
assert!(
|
||||
execution.state().live.is_none(),
|
||||
"the closed root was retained"
|
||||
);
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
@@ -10642,15 +10830,16 @@ fn aggregate_fold_state_is_released_after_input() {
|
||||
// The root is the aggregate the tree built: its fold holds one group and has not run. The
|
||||
// borrow ends before the close below takes the state again.
|
||||
{
|
||||
let RuntimeOp::Aggregate { aggregate, .. } = &execution.state().root.op else {
|
||||
let RuntimeOp::Aggregate { aggregate, .. } =
|
||||
&execution.state().live.as_ref().unwrap().root.op
|
||||
else {
|
||||
panic!("the root is not the aggregate the tree built");
|
||||
};
|
||||
let aggregate = aggregate.as_ref().expect("the root holds an aggregate");
|
||||
assert_eq!(aggregate.groups(), 1);
|
||||
assert!(!aggregate.computed);
|
||||
}
|
||||
assert!(pool.reserved() > 0);
|
||||
execution.state.close().unwrap();
|
||||
execution.finish().unwrap();
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
@@ -11136,6 +11325,265 @@ fn parameterized_topk_reserves_the_values_of_its_parameter() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A consumed dynamic parameter drops charged scalar/broadcast state before the input is finished.
|
||||
#[test]
|
||||
fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() {
|
||||
for histogram in [false, true] {
|
||||
let grid = grid(1_000, 1_000, 2);
|
||||
let input = evaluated_input(Arc::clone(&grid));
|
||||
// Both sides are varying scalars: the broadcast retains one scalar series and emitted
|
||||
// label history after producing the parameter.
|
||||
let parameter = scalar_binary(
|
||||
eval_time(None, Arc::clone(&grid)),
|
||||
eval_time(None, Arc::clone(&grid)),
|
||||
binary_matching(BinaryOp::Add),
|
||||
);
|
||||
let root = NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: if histogram {
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)),
|
||||
parameters: vec![parameter],
|
||||
}
|
||||
} else {
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
}
|
||||
},
|
||||
};
|
||||
let tree = NativeTree::try_new(root).unwrap();
|
||||
let pool = pool();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let metrics = metrics_of(&execution);
|
||||
let parameter_name = if histogram {
|
||||
"root.parameter0"
|
||||
} else {
|
||||
"root.parameter"
|
||||
};
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
let root = &execution.state().live.as_ref().unwrap().root;
|
||||
match &root.op {
|
||||
RuntimeOp::ParameterizedTopK {
|
||||
parameter,
|
||||
vector,
|
||||
state,
|
||||
..
|
||||
} => {
|
||||
assert!(parameter.is_none());
|
||||
assert_eq!(
|
||||
vector.parameter.as_deref(),
|
||||
Some([Some(2.0), Some(4.0)].as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
state.buffer.reservation.size(),
|
||||
2 * std::mem::size_of::<Option<f64>>()
|
||||
);
|
||||
}
|
||||
RuntimeOp::ParameterizedHistogram {
|
||||
parameters,
|
||||
values,
|
||||
state,
|
||||
..
|
||||
} => {
|
||||
assert!(parameters[0].is_none());
|
||||
assert_eq!(
|
||||
values[0].as_deref(),
|
||||
Some([Some(2.0), Some(4.0)].as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
state.buffer.reservation.size(),
|
||||
2 * std::mem::size_of::<Option<f64>>()
|
||||
);
|
||||
}
|
||||
_ => panic!("expected dynamic operator"),
|
||||
}
|
||||
assert_eq!(execution.state().reserved_bytes(), pool.reserved());
|
||||
assert_eq!(pool.reserved(), 2 * std::mem::size_of::<Option<f64>>());
|
||||
assert_eq!(
|
||||
gauge(
|
||||
&metrics,
|
||||
&format!("native.{parameter_name}.current_reserved_bytes")
|
||||
),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
gauge(
|
||||
&metrics,
|
||||
&format!("native.p0.{parameter_name}.peak_reserved_bytes")
|
||||
) > 0
|
||||
);
|
||||
assert_eq!(
|
||||
count(&metrics, parameter_name, NodeMetric::OUTPUT_BATCHES),
|
||||
1
|
||||
);
|
||||
assert!(execution.finish().unwrap().is_empty());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// `scalar()` releases its own retained input after producing, but a map below it still holds
|
||||
/// charged emitted-label history. Consuming the parameter must drop that subtree as well.
|
||||
#[test]
|
||||
fn dynamic_parameters_drop_scalar_of_map_history_before_terminal_close() {
|
||||
for histogram in [false, true] {
|
||||
let grid = grid(0, 1_000, 2);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let input = evaluated_input(Arc::clone(&grid));
|
||||
let parameter = scalar_of(map(
|
||||
evaluated_input_with_source_index(Arc::clone(&grid), 1),
|
||||
vec![MathOp::Abs],
|
||||
));
|
||||
let root = NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: if histogram {
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)),
|
||||
parameters: vec![parameter],
|
||||
}
|
||||
} else {
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
}
|
||||
},
|
||||
};
|
||||
let tree = NativeTree::try_new(root).unwrap();
|
||||
let pool = pool();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let metrics = metrics_of(&execution);
|
||||
let parameter_name = if histogram {
|
||||
"root.parameter0"
|
||||
} else {
|
||||
"root.parameter"
|
||||
};
|
||||
let map_name = format!("{parameter_name}.input");
|
||||
execution
|
||||
.push_evaluated(
|
||||
InputId::new(1),
|
||||
evaluated_chunk(&schema, 0, vec![1.0, 1.0], true),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
let map_current = format!("native.{map_name}.current_reserved_bytes");
|
||||
assert!(
|
||||
gauge(&metrics, &map_current) > 0,
|
||||
"map did not retain emitted history"
|
||||
);
|
||||
// Drive exactly the finishing stage that produces the scalar, before the executor closes.
|
||||
assert!(execution.live_mut().unwrap().pull(true).unwrap().is_none());
|
||||
assert_eq!(gauge(&metrics, &map_current), 0);
|
||||
assert!(
|
||||
gauge(
|
||||
&metrics,
|
||||
&format!("native.p0.{map_name}.peak_reserved_bytes")
|
||||
) > 0
|
||||
);
|
||||
assert_eq!(count(&metrics, &map_name, NodeMetric::OUTPUT_BATCHES), 1);
|
||||
let root = &execution.state().live.as_ref().unwrap().root;
|
||||
match &root.op {
|
||||
RuntimeOp::ParameterizedTopK { parameter, .. } => assert!(parameter.is_none()),
|
||||
RuntimeOp::ParameterizedHistogram { parameters, .. } => {
|
||||
assert!(parameters[0].is_none())
|
||||
}
|
||||
_ => panic!("expected dynamic operator"),
|
||||
}
|
||||
assert_eq!(execution.state().reserved_bytes(), pool.reserved());
|
||||
execution.live_mut().unwrap().root.finish_windows().unwrap();
|
||||
drop(execution);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// A settled parameter with an open window must report the window's finish error before dropping it.
|
||||
#[test]
|
||||
fn dynamic_parameters_finish_windows_before_dropping_the_subtree() {
|
||||
for histogram in [false, true] {
|
||||
let grid = grid(0, 1_000, 2);
|
||||
let input = evaluated_input(Arc::clone(&grid));
|
||||
let parameter_window = window(raw_input_of(1), Arc::clone(&grid));
|
||||
let parameter = scalar_of(parameter_window);
|
||||
let root = NativeNode {
|
||||
output: input.output.clone(),
|
||||
expr: if histogram {
|
||||
NativeExpr::ParameterizedHistogram {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)),
|
||||
parameters: vec![parameter],
|
||||
}
|
||||
} else {
|
||||
NativeExpr::ParameterizedTopK {
|
||||
input: Box::new(input),
|
||||
op: BoundVectorOp::TopK {
|
||||
k: 0.0,
|
||||
top: true,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
parameter: Box::new(parameter),
|
||||
}
|
||||
},
|
||||
};
|
||||
let tree = NativeTree::try_new(root).unwrap();
|
||||
let pool = pool();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool));
|
||||
let live = execution.state.live.as_mut().unwrap();
|
||||
let parameter = match &mut live.root.op {
|
||||
RuntimeOp::ParameterizedTopK { parameter, .. } => parameter.as_deref_mut().unwrap(),
|
||||
RuntimeOp::ParameterizedHistogram { parameters, .. } => parameters[0].as_mut().unwrap(),
|
||||
_ => panic!("expected dynamic operator"),
|
||||
};
|
||||
let RuntimeOp::ScalarOf {
|
||||
input,
|
||||
produced,
|
||||
state,
|
||||
..
|
||||
} = &mut parameter.op
|
||||
else {
|
||||
panic!("expected scalar() parameter");
|
||||
};
|
||||
let RuntimeOp::Window { window, .. } = &mut input.op else {
|
||||
panic!("expected parameter window");
|
||||
};
|
||||
// Force a queued scalar and an open evaluator, an otherwise unreachable invariant break,
|
||||
// so the real finish failure cannot be bypassed by ordinary EOF checks.
|
||||
let chunk = raw_chunk(&raw_schema(), &[0], &[1.0], false);
|
||||
drop(window.as_mut().unwrap().push(&chunk).unwrap());
|
||||
let bytes = grid_chunk_bytes(&grid).unwrap();
|
||||
state.buffer.grow(bytes).unwrap();
|
||||
state.queued.push_back((
|
||||
NativeChunk::Evaluated(super::scalar_chunk(Arc::clone(&grid), 1.0).unwrap()),
|
||||
bytes,
|
||||
));
|
||||
*produced = true;
|
||||
execution.inputs_finished = true;
|
||||
let error = execution.next_chunk().unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("ended before the last chunk"),
|
||||
"{error}"
|
||||
);
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("native.window at root.parameter"),
|
||||
"{error}"
|
||||
);
|
||||
assert!(execution.state().live.is_none());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// A parameterized `topk` released before it computes gives back everything it holds: the input it
|
||||
/// retained, and with it the samples and the label allocations those chunks keep alive.
|
||||
///
|
||||
@@ -11191,7 +11639,7 @@ fn a_parameterized_topk_released_before_it_computes_drops_its_retained_input() {
|
||||
pool.reserved() > 0,
|
||||
"the retained chunk was not charged for"
|
||||
);
|
||||
execution.state.close().unwrap();
|
||||
drop(execution);
|
||||
assert_eq!(pool.reserved(), 0, "the reservation was not returned");
|
||||
assert_eq!(
|
||||
Arc::strong_count(&job),
|
||||
@@ -11311,8 +11759,7 @@ fn parameterized_histogram_hands_over_its_results_from_its_own_queue() {
|
||||
("host=b".to_string(), vec![(0, Some(1.5)), (1, Some(1.5))]),
|
||||
]
|
||||
);
|
||||
// The queue is drained, so what is left is the values the parameters were read into.
|
||||
execution.state.close().unwrap();
|
||||
// The terminal pull dropped the queue and parameter values with the live tree.
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
@@ -11936,7 +12383,7 @@ fn scalar_of_reports_what_it_holds_while_its_input_is_retained() {
|
||||
pool.reserved(),
|
||||
"the reported reservation is not what the node holds"
|
||||
);
|
||||
execution.state.close().unwrap();
|
||||
drop(execution);
|
||||
assert_eq!(pool.reserved(), 0, "the reservation was not returned");
|
||||
}
|
||||
|
||||
@@ -12559,11 +13006,9 @@ fn a_label_change_admits_the_label_it_writes_before_building_it() {
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// An ended execution keeps its tree and its registered counters, but neither the input schema nor
|
||||
/// the grid it is addressed on. The leaf itself stays -- the root still says which routing slot it
|
||||
/// is, and the handles the caller registered before the run still read what the nodes recorded --
|
||||
/// but the slot that fed it is taken. Both ways an execution ends are covered: the answer is drained,
|
||||
/// and a rejected push stops it first.
|
||||
/// An ended execution drops its tree, input schema and grid, while registered counters survive.
|
||||
/// Both ways an execution ends are covered: the answer is drained, and a rejected push stops it
|
||||
/// first.
|
||||
#[test]
|
||||
fn an_ended_execution_keeps_no_input_schema_or_grid() {
|
||||
for fails in [false, true] {
|
||||
@@ -12642,13 +13087,8 @@ fn an_ended_execution_keeps_no_input_schema_or_grid() {
|
||||
assert_eq!(execution.state().reserved_bytes(), 0);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
assert!(
|
||||
execution.state().inputs.is_empty(),
|
||||
"the ended execution kept an input slot"
|
||||
);
|
||||
assert_eq!(
|
||||
execution.state().inputs.capacity(),
|
||||
0,
|
||||
"the ended execution kept the slot vector's backing"
|
||||
execution.state().live.is_none(),
|
||||
"the ended execution kept its input slots"
|
||||
);
|
||||
let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES);
|
||||
assert_eq!(
|
||||
@@ -12662,8 +13102,8 @@ fn an_ended_execution_keeps_no_input_schema_or_grid() {
|
||||
"the peak the leaf reached did not survive the end of the execution"
|
||||
);
|
||||
assert!(
|
||||
root_holds(&execution, holds_input),
|
||||
"the ended execution replaced the root it was built from"
|
||||
!root_holds(&execution, holds_input),
|
||||
"the ended execution kept its root"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user