From ae9fb131e2fb17b9deeb0c315ced49a288dd07d9 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Nov 2026 13:58:56 +0800 Subject: [PATCH] refactor(promql): drop native runtime state at terminal (#10242) Signed-off-by: discord9 --- src/promql/src/native/execute.rs | 676 ++++++++----------------- src/promql/src/native/execute/tests.rs | 628 +++++++++++++++++++---- 2 files changed, 748 insertions(+), 556 deletions(-) diff --git a/src/promql/src/native/execute.rs b/src/promql/src/native/execute.rs index cf3057340e1..4945dfa9254 100644 --- a/src/promql/src/native/execute.rs +++ b/src/promql/src/native/execute.rs @@ -526,9 +526,10 @@ impl NodeBuffer { fn record(&self) { self.metrics.record_reserved_bytes(self.reservation.size()); } +} - /// Returns every byte this node holds to the pool. - fn release(&mut self) { +impl Drop for NodeBuffer { + fn drop(&mut self) { self.reservation.free(); self.record(); } @@ -1588,7 +1589,7 @@ impl BinaryRuntime { self.operands .get(side) .map(|operand| operand.pending(chunk)) - .ok_or_else(no_binary) + .ok_or_else(invalid_binary_operand) } /// Reports an operand that ended inside a series. @@ -1609,7 +1610,7 @@ impl BinaryRuntime { // number of fields, or fewer samples than its axis places rows. The kernel's own check is // run here, before the chunk enters the buffer, and the side is added to what it reports. let Some(schema) = self.schemas.get(side) else { - return Err(no_binary()); + return Err(invalid_binary_operand()); }; validate_binary_operand(schema.as_ref(), std::slice::from_ref(&chunk.series)).map_err( |error| { @@ -1622,7 +1623,7 @@ impl BinaryRuntime { )?; let reader = self.reader(side); let Some(operand) = self.operands.get_mut(side) else { - return Err(no_binary()); + return Err(invalid_binary_operand()); }; operand.retain(chunk, reader) } @@ -2784,6 +2785,7 @@ struct RuntimeNode { op: RuntimeOp, } +// Each payload precedes its NodeState, so it drops before that node's reservation. enum RuntimeOp { Input(InputId), Scalar, @@ -2813,7 +2815,7 @@ enum RuntimeOp { }, Instant { input: Box, - instant: Option, + instant: InstantRuntime, state: NodeState, }, Merge { @@ -2823,59 +2825,62 @@ enum RuntimeOp { }, Window { input: Box, + // `finish` consumes the evaluator; it is taken only during the terminal finish pass. window: Option, merge: Option, state: NodeState, }, Map { input: Box, - map: Option, + map: MapRuntime, state: NodeState, }, ParameterizedMap { input: Box, parameters: Vec, - map: Option, + map: ParameterizedMapRuntime, state: NodeState, }, Timestamp { input: Box, - schema: Option>, + schema: Arc, state: NodeState, }, Binary { left: Box, right: Box, - binary: Option, + binary: BinaryRuntime, state: NodeState, }, Broadcast { left: Box, right: Box, - broadcast: Option, + broadcast: BroadcastRuntime, state: NodeState, }, Aggregate { input: Box, - aggregate: Option, + aggregate: AggregateRuntime, state: NodeState, }, Vector { input: Box, - vector: Option, + vector: VectorRuntime, state: NodeState, }, ParameterizedTopK { input: Box, - parameter: Box, - vector: Option, + // Taken and dropped after its scalar values are installed. + parameter: Option>, + vector: VectorRuntime, state: NodeState, }, ParameterizedHistogram { input: Box, - parameters: Vec, + // Each parameter is taken and dropped after its scalar values are installed. + parameters: Vec>, literals: Vec>, - vector: Option, + vector: VectorRuntime, values: Vec>>>, parameter_bytes: Vec, state: NodeState, @@ -2883,6 +2888,8 @@ enum RuntimeOp { } /// State shared by the stateful operators. +/// +/// The queue and emitted history must precede the buffer so they drop before its reservation. struct NodeState { queued: VecDeque<(NativeChunk, usize)>, queued_tables: usize, @@ -2944,17 +2951,17 @@ struct InputSlot { state: NodeState, } -/// State for one execution of a bound tree. -/// -/// The state owns its runtime tree outright: the topology is the one the compiled tree built, and -/// nothing outside reaches into it, so a descent and the release that follows a failure are both -/// borrows of this one owner. +/// State for one execution of a bound tree. The live tree and input slots are dropped together +/// when the executor finishes or fails; the public state remains available for inspection. pub struct NativeExecutionState { + live: Option, +} + +struct LiveExecution { root: RuntimeNode, inputs: Vec, /// A constant root waiting for its only pull; operand scalars remain chunkless. scalar_root: Option<(Arc, f64, NodeState)>, - annotations: Option, } impl NodeState { @@ -3115,7 +3122,7 @@ impl RuntimeNode { ( RuntimeOp::Instant { input: Box::new(child), - instant: Some(InstantRuntime::try_new(&input_schema, op)?), + instant: InstantRuntime::try_new(&input_schema, op)?, state: { let mut state = NodeState::new( format!("native.instant at {location}"), @@ -3217,10 +3224,10 @@ impl RuntimeNode { ( RuntimeOp::Map { input: Box::new(child), - map: Some(MapRuntime { + map: MapRuntime { ops: Arc::clone(&op.ops), schema, - }), + }, state, }, true, @@ -3274,13 +3281,13 @@ impl RuntimeNode { RuntimeOp::ParameterizedMap { input: Box::new(child), parameters: nodes, - map: Some(ParameterizedMapRuntime { + map: ParameterizedMapRuntime { op: *op, schema, literals, values: vec![None; parameters.len()], parameter_bytes: 0, - }), + }, state, }, true, @@ -3307,7 +3314,7 @@ impl RuntimeNode { ( RuntimeOp::Timestamp { input: Box::new(child), - schema: Some(schema), + schema, state, }, true, @@ -3359,7 +3366,7 @@ impl RuntimeNode { RuntimeOp::Binary { left: Box::new(left_node), right: Box::new(right_node), - binary: Some(binary), + binary, state: NodeState::new( format!("native.binary at {location}"), Arc::clone(&metrics), @@ -3387,7 +3394,7 @@ impl RuntimeNode { RuntimeOp::Broadcast { left: Box::new(left_node), right: Box::new(right_node), - broadcast: Some(broadcast), + broadcast, state, }, shared, @@ -3437,7 +3444,7 @@ impl RuntimeNode { RuntimeOp::Binary { left: Box::new(left_node), right: Box::new(right_node), - binary: Some(set), + binary: set, state: NodeState::new( format!("native.set at {location}"), Arc::clone(&metrics), @@ -3488,7 +3495,7 @@ impl RuntimeNode { RuntimeOp::Binary { left: Box::new(base_node), right: Box::new(info_node), - binary: Some(runtime), + binary: runtime, state: NodeState::new( format!("native.info at {location}"), Arc::clone(&metrics), @@ -3512,11 +3519,7 @@ impl RuntimeNode { Arc::clone(&metrics), pool, ); - let aggregate = Some(AggregateRuntime::new( - op.clone(), - schema, - &mut state.buffer, - )?); + let aggregate = AggregateRuntime::new(op.clone(), schema, &mut state.buffer)?; let (child, _) = Self::build( input, pool, @@ -3573,8 +3576,8 @@ impl RuntimeNode { ( RuntimeOp::ParameterizedTopK { input: Box::new(child), - parameter: Box::new(parameter), - vector: Some(vector), + parameter: Some(Box::new(parameter)), + vector, state, }, // `topk` keeps whole input series and canonicalises no label set, so a dynamic @@ -3635,7 +3638,7 @@ impl RuntimeNode { format!("{location}.parameter{index}"), duplicate_evidence.clone(), )?; - built.push(runtime); + built.push(Some(runtime)); } ( RuntimeOp::ParameterizedHistogram { @@ -3644,7 +3647,7 @@ impl RuntimeNode { values: vec![None; built.len()], parameters: built, literals, - vector: Some(vector), + vector, state: NodeState::new( format!("native.vector at {location}"), Arc::clone(&metrics), @@ -3695,7 +3698,7 @@ impl RuntimeNode { ( RuntimeOp::Vector { input: Box::new(child), - vector: Some(vector), + vector, state: NodeState::new( format!("native.vector at {location}"), Arc::clone(&metrics), @@ -3734,14 +3737,18 @@ impl RuntimeNode { input, parameter, .. } => { input.register_metrics(&format!("{name}.input"), set, partition); - parameter.register_metrics(&format!("{name}.parameter"), set, partition); + if let Some(parameter) = parameter { + parameter.register_metrics(&format!("{name}.parameter"), set, partition); + } } RuntimeOp::ParameterizedHistogram { input, parameters, .. } => { input.register_metrics(&format!("{name}.input"), set, partition); for (i, parameter) in parameters.iter().enumerate() { - parameter.register_metrics(&format!("{name}.parameter{i}"), set, partition); + if let Some(parameter) = parameter { + parameter.register_metrics(&format!("{name}.parameter{i}"), set, partition); + } } } RuntimeOp::ParameterizedMap { @@ -3800,7 +3807,7 @@ impl RuntimeNode { .reservation .size() .saturating_add(input.reserved_bytes()), - |n, p| n.saturating_add(p.reserved_bytes()), + |n, p| n.saturating_add(p.as_ref().map_or(0, RuntimeNode::reserved_bytes)), ), RuntimeOp::ParameterizedTopK { input, @@ -3812,7 +3819,7 @@ impl RuntimeNode { .reservation .size() .saturating_add(input.reserved_bytes()) - .saturating_add(parameter.reserved_bytes()), + .saturating_add(parameter.as_ref().map_or(0, |p| p.reserved_bytes())), RuntimeOp::ParameterizedMap { input, parameters, @@ -3870,18 +3877,14 @@ impl RuntimeNode { broadcast, .. } => { - if let Some(broadcast) = broadcast { - broadcast.annotations = Some(collector.clone()); - } + broadcast.annotations = Some(collector.clone()); left.set_annotations(collector.clone()); right.set_annotations(collector); } RuntimeOp::Aggregate { input, .. } => input.set_annotations(collector), RuntimeOp::Timestamp { input, .. } => input.set_annotations(collector), RuntimeOp::Vector { input, vector, .. } => { - if let Some(vector) = vector { - vector.annotations = Some(collector.clone()); - } + vector.annotations = Some(collector.clone()); input.set_annotations(collector); } // `scalar()` answers a value of its own and records nothing, so only its input needs the @@ -3894,11 +3897,11 @@ impl RuntimeNode { vector, .. } => { - if let Some(vector) = vector { - vector.annotations = Some(collector.clone()); - } + vector.annotations = Some(collector.clone()); input.set_annotations(collector.clone()); - parameter.set_annotations(collector); + if let Some(parameter) = parameter { + parameter.set_annotations(collector); + } } // The histogram kernel reports a native histogram and classic buckets of one label set // through the collector, so the runtime it runs on needs it as well as its children. @@ -3908,11 +3911,9 @@ impl RuntimeNode { vector, .. } => { - if let Some(vector) = vector { - vector.annotations = Some(collector.clone()); - } + vector.annotations = Some(collector.clone()); input.set_annotations(collector.clone()); - for parameter in parameters { + for parameter in parameters.iter_mut().flatten() { parameter.set_annotations(collector.clone()); } } @@ -4103,9 +4104,7 @@ impl RuntimeNode { instant, state, } => { - let Some(selection) = instant.as_mut() else { - return Err(no_instant()); - }; + let selection = instant; // A selection decides a whole series at once, so every chunk before the one that // closes it yields nothing and the selection asks its input again. loop { @@ -4173,9 +4172,7 @@ impl RuntimeNode { } } RuntimeOp::Map { input, map, state } => { - let Some(operation) = map.as_ref() else { - return Err(no_map()); - }; + let operation = map; // An operation that empties every position yields no chunk, and the map asks its // input again rather than reporting a decision it did not make. loop { @@ -4201,9 +4198,7 @@ impl RuntimeNode { map, state, } => { - let Some(operation) = map.as_mut() else { - return Err(no_map()); - }; + let operation = map; // Every parameter is read whole first, as a broadcast reads its scalar; a constant // one is materialized from its binding. for (index, parameter) in parameters.iter_mut().enumerate() { @@ -4271,36 +4266,29 @@ impl RuntimeNode { input, schema, state, - } => { - let schema = schema.as_ref().ok_or_else(|| { - DataFusionError::Execution("timestamp schema was released".into()) - })?; - loop { - let Some(chunk) = input.pull(inputs, inputs_finished)? else { - return Ok(None); - }; - let NativeChunk::Evaluated(chunk) = chunk else { - return Err(DataFusionError::Execution( - "timestamp consumes evaluated samples, not raw samples".into(), - )); - }; - if let Some(output) = state - .timestamp(schema, chunk) - .map_err(|error| state.context(error))? - { - return Ok(Some(NativeChunk::Evaluated(output))); - } + } => loop { + let Some(chunk) = input.pull(inputs, inputs_finished)? else { + return Ok(None); + }; + let NativeChunk::Evaluated(chunk) = chunk else { + return Err(DataFusionError::Execution( + "timestamp consumes evaluated samples, not raw samples".into(), + )); + }; + if let Some(output) = state + .timestamp(schema, chunk) + .map_err(|error| state.context(error))? + { + return Ok(Some(NativeChunk::Evaluated(output))); } - } + }, RuntimeOp::Binary { left, right, binary, state, } => { - let Some(operation) = binary.as_mut() else { - return Err(no_binary()); - }; + let operation = binary; // Both operands are read to the end of what they hand over: before the input ends a // binary node only retains the chunks, because a pair is not decided until both // sides are whole, and after it ends this is the drain that empties the two. @@ -4333,9 +4321,7 @@ impl RuntimeNode { broadcast, state, } => { - let Some(operation) = broadcast.as_mut() else { - return Err(no_broadcast()); - }; + let operation = broadcast; // A constant is on the node's binding, so only the vector side is pulled. A varying // scalar is read first, whole; then each vector chunk is transformed as it arrives // and handed up, which is what keeps a broadcast over a whole vector from costing @@ -4383,9 +4369,7 @@ impl RuntimeNode { aggregate, state, } => { - let Some(fold) = aggregate.as_mut() else { - return Err(no_aggregate()); - }; + let fold = aggregate; // What the fold reads is not retained -- a chunk is folded as it arrives and // dropped -- so reading the child dry costs the fold's states and not the input's // bytes. Nothing is answered before the input ends: until then a group's value at a @@ -4411,10 +4395,13 @@ impl RuntimeNode { vector, state, } => { - let operation = vector.as_mut().ok_or_else(no_vector)?; + let operation = vector; if operation.parameter.is_none() { let mut slot = None; - while let Some(chunk) = parameter.pull(inputs, inputs_finished)? { + let parameter_node = parameter.as_mut().ok_or_else(|| { + DataFusionError::Internal("topk parameter already consumed".into()) + })?; + while let Some(chunk) = parameter_node.pull(inputs, inputs_finished)? { state .hold_scalar(&mut operation.parameter_bytes, &mut slot, chunk) .map_err(|error| state.context(error))?; @@ -4423,7 +4410,7 @@ impl RuntimeNode { // A parameter that folds its own input (a `scalar()` over a selector) has // nothing to hand over until that input ends, so its absence is a reason to // be pulled again rather than a failure. - if !parameter.settled(inputs) { + if !parameter_node.settled(inputs) { return Ok(None); } slot.ok_or_else(|| { @@ -4468,7 +4455,10 @@ impl RuntimeNode { state .settle_vector(operation) .map_err(|error| state.context(error))?; - parameter.release(); + // Finish windows before dropping the consumed subtree; its children may still + // hold charged emitted history or scalar series even after the scalar was read. + parameter_node.finish_windows()?; + *parameter = None; } while let Some(chunk) = input.pull(inputs, inputs_finished)? { state @@ -4492,7 +4482,7 @@ impl RuntimeNode { parameter_bytes, state, } => { - let operation = vector.as_mut().ok_or_else(no_vector)?; + let operation = vector; for index in 0..parameters.len() { if values[index].is_none() { let mut slot = None; @@ -4507,12 +4497,15 @@ impl RuntimeNode { value, )?; } else { - while let Some(chunk) = - parameters[index].pull(inputs, inputs_finished)? - { + let parameter = parameters[index].as_mut().ok_or_else(|| { + DataFusionError::Internal( + "histogram parameter already consumed".into(), + ) + })?; + while let Some(chunk) = parameter.pull(inputs, inputs_finished)? { state.hold_scalar(&mut parameter_bytes[index], &mut slot, chunk)?; } - if !parameters[index].settled(inputs) { + if !parameter.settled(inputs) { return Ok(None); } } @@ -4561,7 +4554,10 @@ impl RuntimeNode { .copied() .fold(0usize, usize::saturating_add); state.settle_vector(operation)?; - parameters[index].release(); + if let Some(parameter) = &mut parameters[index] { + parameter.finish_windows()?; + } + parameters[index] = None; } } // Nothing is settled here: what this node holds changes as a parameter is converted @@ -4591,9 +4587,7 @@ impl RuntimeNode { vector, state, } => { - let Some(operation) = vector.as_mut() else { - return Err(no_vector()); - }; + let operation = vector; // Which samples are kept at a point is not decided until the whole input has // arrived, so the input is read to the end before anything is handed over. while let Some(chunk) = input.pull(inputs, inputs_finished)? { @@ -4644,11 +4638,10 @@ impl RuntimeNode { RuntimeOp::ParameterizedMap { input, parameters, - map, state, + .. } => { state.queued.is_empty() - && map.is_some() && input.settled(inputs) && parameters.iter().all(|parameter| parameter.settled(inputs)) } @@ -4660,9 +4653,7 @@ impl RuntimeNode { } => { // Parameter nodes are released after their values are installed; `computed` // already implies those values were consumed. - state.queued.is_empty() - && vector.as_ref().is_some_and(|v| v.computed) - && input.settled(inputs) + state.queued.is_empty() && vector.computed && input.settled(inputs) } RuntimeOp::ParameterizedTopK { input, @@ -4672,9 +4663,7 @@ impl RuntimeNode { } => { // Parameter nodes are released after their values are installed; `computed` // already implies the parameter was consumed. - state.queued.is_empty() - && vector.as_ref().is_some_and(|v| v.computed) - && input.settled(inputs) + state.queued.is_empty() && vector.computed && input.settled(inputs) } RuntimeOp::VectorOf { input, @@ -4712,20 +4701,12 @@ impl RuntimeNode { input, aggregate, state, - } => { - state.queued.is_empty() - && aggregate.as_ref().is_some_and(|fold| fold.computed) - && input.settled(inputs) - } + } => state.queued.is_empty() && aggregate.computed && input.settled(inputs), RuntimeOp::Vector { input, vector, state, - } => { - state.queued.is_empty() - && vector.as_ref().is_some_and(|operation| operation.computed) - && input.settled(inputs) - } + } => state.queued.is_empty() && vector.computed && input.settled(inputs), // A vector match is settled once its kernel ran and both operands are: what one side // answers at a point depends on the other side. RuntimeOp::Binary { @@ -4735,23 +4716,15 @@ impl RuntimeNode { state, } => { state.queued.is_empty() - && binary.as_ref().is_some_and(|operation| operation.computed) + && binary.computed && left.settled(inputs) && right.settled(inputs) } // A broadcast owes what its sides have not handed over yet. A constant side is settled // from the start. RuntimeOp::Broadcast { - left, - right, - broadcast, - state, - } => { - state.queued.is_empty() - && broadcast.is_some() - && left.settled(inputs) - && right.settled(inputs) - } + left, right, state, .. + } => state.queued.is_empty() && left.settled(inputs) && right.settled(inputs), } } @@ -4785,14 +4758,20 @@ impl RuntimeNode { input, parameters, .. } => { let mut flushed = input.flush_ready_merges(inputs)?; - for p in parameters { + for p in parameters.iter_mut().flatten() { flushed |= p.flush_ready_merges(inputs)?; } Ok(flushed) } RuntimeOp::ParameterizedTopK { input, parameter, .. - } => Ok(input.flush_ready_merges(inputs)? | parameter.flush_ready_merges(inputs)?), + } => { + let mut flushed = input.flush_ready_merges(inputs)?; + if let Some(parameter) = parameter { + flushed |= parameter.flush_ready_merges(inputs)?; + } + Ok(flushed) + } RuntimeOp::Binary { left, right, .. } | RuntimeOp::Broadcast { left, right, .. } => { let left_flushed = left.flush_ready_merges(inputs)?; let right_flushed = right.flush_ready_merges(inputs)?; @@ -4851,10 +4830,19 @@ impl RuntimeNode { | RuntimeOp::VectorOf { input, .. } => input.any_merge_pending(), RuntimeOp::ParameterizedHistogram { input, parameters, .. - } => input.any_merge_pending() || parameters.iter().any(RuntimeNode::any_merge_pending), + } => { + input.any_merge_pending() + || parameters + .iter() + .flatten() + .any(RuntimeNode::any_merge_pending) + } RuntimeOp::ParameterizedTopK { input, parameter, .. - } => input.any_merge_pending() || parameter.any_merge_pending(), + } => { + input.any_merge_pending() + || parameter.as_ref().is_some_and(|p| p.any_merge_pending()) + } RuntimeOp::Binary { left, right, .. } | RuntimeOp::Broadcast { left, right, .. } => { left.any_merge_pending() || right.any_merge_pending() } @@ -4903,13 +4891,15 @@ impl RuntimeNode { vector, state, } => { - if let Some(v) = vector { - v.input - .ensure_closed(Reader::Vector) - .map_err(|e| state.context(e))?; - } + vector + .input + .ensure_closed(Reader::Vector) + .map_err(|e| state.context(e))?; input.unclosed()?; - parameter.unclosed() + if let Some(parameter) = parameter { + parameter.unclosed()?; + } + Ok(()) } RuntimeOp::ParameterizedHistogram { input, @@ -4918,13 +4908,15 @@ impl RuntimeNode { state, .. } => { - if let Some(v) = vector { - v.input - .ensure_closed(Reader::Vector) - .map_err(|e| state.context(e))?; - } + vector + .input + .ensure_closed(Reader::Vector) + .map_err(|e| state.context(e))?; input.unclosed()?; - parameters.iter().try_for_each(RuntimeNode::unclosed) + parameters + .iter() + .flatten() + .try_for_each(RuntimeNode::unclosed) } RuntimeOp::Binary { left, @@ -4932,9 +4924,7 @@ impl RuntimeNode { binary, state, } => { - if let Some(operation) = binary { - operation.closed().map_err(|error| state.context(error))?; - } + binary.closed().map_err(|error| state.context(error))?; left.unclosed()?; right.unclosed() } @@ -4947,12 +4937,10 @@ impl RuntimeNode { vector, state, } => { - if let Some(operation) = vector { - operation - .input - .ensure_closed(Reader::Vector) - .map_err(|error| state.context(error))?; - } + vector + .input + .ensure_closed(Reader::Vector) + .map_err(|error| state.context(error))?; input.unclosed() } } @@ -4987,13 +4975,18 @@ impl RuntimeNode { | RuntimeOp::Merge { input, .. } => input.finish_windows(), RuntimeOp::ParameterizedTopK { input, parameter, .. - } => input.finish_windows().and(parameter.finish_windows()), + } => { + let first = input.finish_windows(); + let parameter = parameter.as_mut().map_or(Ok(()), |p| p.finish_windows()); + first.and(parameter) + } RuntimeOp::ParameterizedHistogram { input, parameters, .. } => { let first = input.finish_windows(); parameters .iter_mut() + .flatten() .map(RuntimeNode::finish_windows) .fold(first, Result::and) } @@ -5019,165 +5012,6 @@ impl RuntimeNode { } } } - - /// Gives back everything this node and its children hold, keeping the topology. - /// - /// The operation's payload goes first: it is what the reservation was taken for, and dropping - /// it after the reservation went back would hand the pool bytes it cannot admit while they are - /// still read. A released node keeps its shape — an input keeps its routing slot, a binary both - /// of its operands — because what a node is does not change when the bytes it held go back. - fn release(&mut self) { - match &mut self.op { - // A source keeps its queue in the caller-owned slot, which is released beside the tree, - // and a scalar holds nothing at all. - RuntimeOp::Input(_) | RuntimeOp::Scalar => {} - RuntimeOp::EvalTime { state, .. } => state.release(), - RuntimeOp::ScalarOf { - input, - retained, - state, - .. - } => { - retained.release(); - state.release(); - input.release(); - } - RuntimeOp::ParameterizedMap { - input, - parameters, - map, - state, - } => { - *map = None; - state.release(); - input.release(); - for parameter in parameters { - parameter.release(); - } - } - RuntimeOp::ParameterizedTopK { - input, - parameter, - vector, - state, - } => { - // What the node holds is the input it retained and the values it read the parameter - // into, whether or not the kernel ran; both go before the reservation does, as the - // ordinary vector branch drops its assembled result. - if let Some(mut v) = vector.take() { - v.input.release(); - } - state.release(); - input.release(); - parameter.release(); - } - RuntimeOp::ParameterizedHistogram { - input, - parameters, - vector, - state, - values, - .. - } => { - if let Some(mut v) = vector.take() { - v.input.release(); - } - for parameter in parameters { - parameter.release(); - } - values.clear(); - state.release(); - input.release(); - } - RuntimeOp::VectorOf { input, state, .. } => { - state.release(); - input.release(); - } - RuntimeOp::Instant { - input, - instant, - state, - } => { - *instant = None; - state.release(); - input.release(); - } - RuntimeOp::Merge { - input, - merge, - state, - } => { - *merge = None; - state.release(); - input.release(); - } - RuntimeOp::Window { - input, - window, - merge, - state, - } => { - *window = None; - *merge = None; - state.release(); - input.release(); - } - RuntimeOp::Map { input, map, state } => { - *map = None; - state.release(); - input.release(); - } - RuntimeOp::Timestamp { - input, - schema, - state, - } => { - *schema = None; - state.release(); - input.release(); - } - RuntimeOp::Aggregate { - input, - aggregate, - state, - } => { - *aggregate = None; - state.release(); - input.release(); - } - RuntimeOp::Vector { - input, - vector, - state, - } => { - *vector = None; - state.release(); - input.release(); - } - RuntimeOp::Binary { - left, - right, - binary, - state, - } => { - *binary = None; - state.release(); - left.release(); - right.release(); - } - RuntimeOp::Broadcast { - left, - right, - broadcast, - state, - } => { - *broadcast = None; - state.release(); - left.release(); - right.release(); - } - } - } } impl NativeExecutionState { @@ -5223,17 +5057,17 @@ impl NativeExecutionState { NativeType::AggregateState(_) => {} } Ok(Self { - root, - inputs, - scalar_root, - annotations: None, + live: Some(LiveExecution { + root, + inputs, + scalar_root, + }), }) } pub fn set_annotations(&mut self, collector: Option) { - self.annotations = collector.clone(); - if let Some(collector) = collector { - self.root.set_annotations(collector); + if let (Some(live), Some(collector)) = (&mut self.live, collector) { + live.root.set_annotations(collector); } } @@ -5243,39 +5077,39 @@ impl NativeExecutionState { /// nodes' own counters and gauges, so a later read of it reports what they recorded and keeps /// no topology of this execution alive. pub(crate) fn register_metrics(&self, set: &ExecutionPlanMetricsSet, partition: usize) { - self.root.register_metrics("root", set, partition); + if let Some(live) = &self.live { + live.root.register_metrics("root", set, partition); + } } /// Returns the runtime input slots bound to `source_index`. pub fn input_ids(&self, source_index: usize) -> impl Iterator + '_ { - self.inputs + self.live .iter() + .flat_map(|live| live.inputs.iter()) .enumerate() .filter(move |(_, slot)| slot.source_index == source_index) .map(|(index, _)| InputId::new(index)) } pub fn reserved_bytes(&self) -> usize { - self.inputs - .iter() - .map(|slot| slot.state.buffer.reservation.size()) - .fold(0usize, usize::saturating_add) - .saturating_add(self.root.reserved_bytes()) - .saturating_add( - self.scalar_root - .as_ref() - .map_or(0, |(_, _, state)| state.buffer.reservation.size()), - ) + self.live.as_ref().map_or(0, |live| { + live.inputs + .iter() + .map(|slot| slot.state.buffer.reservation.size()) + .fold(0usize, usize::saturating_add) + .saturating_add(live.root.reserved_bytes()) + .saturating_add( + live.scalar_root + .as_ref() + .map_or(0, |(_, _, state)| state.buffer.reservation.size()), + ) + }) } } -/// What the driver asks of one execution's state. -/// -/// Every operation borrows the tree this state owns — exclusively to pull, shared to inspect it — -/// while it works on the nodes below, and gives the borrow back before returning. Nothing else -/// reaches into the tree, so a failed operation leaves nothing held behind it and the release that -/// follows a failure borrows the same tree again. -impl NativeExecutionState { +/// Pulls and checks the one live tree the execution owns. +impl LiveExecution { /// Pulls the next chunk the root of this state's tree hands over. /// /// `inputs_finished` is the caller's word that no more source chunks will arrive; it is passed @@ -5321,54 +5155,9 @@ impl NativeExecutionState { self.root.any_merge_pending() } - /// Closes the execution: finishes every window's evaluator and gives everything back. - /// - /// The evaluators are finished before anything is released, so the first failure is reported - /// from the state that caused it. Everything is released whether or not one of them failed: - /// the caller reads the error either way, and holding the bytes past it would keep the query's - /// memory in use for as long as the executor is alive. - fn close(&mut self) -> Result<()> { - let finished = self.root.finish_windows(); - self.root.release(); - if let Some((_, _, mut state)) = self.scalar_root.take() { - state.release(); - } - // The slots are taken rather than only released: what a released slot still holds is its - // schema and the grid that schema is addressed on, and a caller that keeps the stopped - // executor would otherwise keep the input's schema and materialized timestamps alive past - // the execution. The leaf itself stays -- the root keeps its `Input(InputId)` and the - // nodes keep their metric handles -- and it holds neither. - for mut slot in std::mem::take(&mut self.inputs) { - slot.release(); - } - finished - } - - /// Gives back everything this execution still holds: its payloads, its slots and the bytes - /// they reserved. - /// - /// The slots are taken here as they are when the execution closes, so the input's schema and - /// grid do not outlive it in an executor its caller still holds; a second call finds an empty - /// vector and nothing left to take. - fn release(&mut self) { - self.root.release(); - if let Some((_, _, mut state)) = self.scalar_root.take() { - state.release(); - } - for mut slot in std::mem::take(&mut self.inputs) { - slot.release(); - } - } -} - -impl Drop for NativeExecutionState { - /// Returns what the execution never gave back itself. - /// - /// The tree goes with this state, so its payloads would be dropped here in any case; the - /// release is what gives the pool their bytes back and leaves the counters reporting what the - /// nodes did rather than what they held. - fn drop(&mut self) { - self.release(); + /// Finishes every window before dropping the live tree and its reservations. + fn close(mut self) -> Result<()> { + self.root.finish_windows() } } @@ -6391,23 +6180,6 @@ impl NodeState { fn emitted_bytes(&self) -> usize { self.emitted.as_ref().map_or(0, Emitted::bytes) } - - /// Returns the shared state this node holds: its queue, its tables and its reservation. - /// - /// The operation's own payload — the evaluator, the retained operands, the fold — is not this - /// state's: it belongs to the node that owns this state, and it is dropped there before this - /// runs. Dropping it first is what keeps the reservation from going back to the pool while the - /// memory it was taken for is still held. - fn release(&mut self) { - // The label sets this node handed out and the chunks it queued go first, for the same - // reason: their memory is what the reservation below returns. - self.emitted = None; - self.queued.clear(); - self.queued.shrink_to_fit(); - self.queued_tables = 0; - // The reservation is returned once, below, for everything this node still held. - self.buffer.release(); - } } impl InputSlot { @@ -6450,16 +6222,6 @@ impl InputSlot { } Ok(Some(chunk)) } - - /// Returns what this input still holds: its open-series flag, its queue and its reservation. - /// - /// A leaf that holds no chunk holds no series either, so nothing it handed over is still open - /// for it: the flag is what closing the execution reads, and it must not survive the state that - /// could have kept a series alive. - fn release(&mut self) { - self.open_source_series = false; - self.state.release(); - } } /// Drives one execution of a bound tree. @@ -6519,6 +6281,14 @@ impl NativeExecutor { &self.state } + fn live(&self) -> Result<&LiveExecution> { + self.state.live.as_ref().ok_or_else(no_live_execution) + } + + fn live_mut(&mut self) -> Result<&mut LiveExecution> { + self.state.live.as_mut().ok_or_else(no_live_execution) + } + /// Queues one raw source chunk at the input `input` names. /// /// A tree of unary nodes has one input; a binary node adds one per operand, so a tree of nested @@ -6617,7 +6387,8 @@ impl NativeExecutor { loop { // The state's tree is the one this execution was built from, and a pull is asked of its // root; the nodes below it read their own children from there. - match self.state.pull(self.inputs_finished) { + let inputs_finished = self.inputs_finished; + match self.live_mut()?.pull(inputs_finished) { Ok(Some(chunk)) => return Ok(Some(chunk)), // Nothing has settled: the stages below decide whether that is a stall or an end. Ok(None) => {} @@ -6637,7 +6408,7 @@ impl NativeExecutor { // series the caller never closed is still open here and nowhere else. It is // refused before a binary node computes: half a vector is not a vector, so the // kernel must not allocate a workspace to match one. - if let Err(error) = self.state.unclosed() { + if let Err(error) = self.live()?.unclosed() { self.stop(&error); return Err(error); } @@ -6649,7 +6420,7 @@ impl NativeExecutor { // first is what keeps the two in order: the window produces, the pull that follows // carries its output to whoever reads it, and only then can a binary node decide // whether both of its operands are complete. - match self.state.flush_ready_merges() { + match self.live_mut()?.flush_ready_merges() { // A window answered: what it queued is pulled on the next turn, and that pull is // also what carries its answer to a node above it. Ok(true) => continue, @@ -6659,7 +6430,7 @@ impl NativeExecutor { return Err(error); } } - if self.state.any_merge_pending() { + if self.live()?.any_merge_pending() { // A window still holds collected input, so something below it has not finished: // pulling again is what gives that node its turn. continue; @@ -6667,7 +6438,8 @@ impl NativeExecutor { // Every node has answered everything it can. Closing them is what hands back the // reservations they were charged, so pulling to exhaustion leaves the pool as it found // it. - return match self.state.close() { + let live = self.state.live.take().ok_or_else(no_live_execution)?; + return match live.close() { Ok(()) => { self.phase = Phase::Finished; Ok(None) @@ -6744,9 +6516,9 @@ impl NativeExecutor { // The id counts the leaves left to right, so an id past the last one names no leaf at all: // a caller that miscounted its tree is refused rather than answered from whichever leaf the // count happened to reach. - let count = self.state.inputs.len(); - let slot = self - .state + let live = self.live_mut()?; + let count = live.inputs.len(); + let slot = live .inputs .get_mut(input.index()) .ok_or_else(|| no_input(count, input))?; @@ -6773,7 +6545,7 @@ impl NativeExecutor { /// here -- its payload, its queue and its reservation -- so a stopped execution holds nothing /// while the caller still owns it. fn stop(&mut self, error: &DataFusionError) { - self.state.release(); + self.state.live = None; self.phase = Phase::Failed(error.to_string()); } } @@ -6963,34 +6735,14 @@ fn no_input(inputs: usize, input: InputId) -> DataFusionError { )) } -/// The error for a node asked to compute without the runtime its kind holds. -fn no_binary() -> DataFusionError { - DataFusionError::Execution("the node holds no binary operation".to_string()) +/// A binary operation was asked to read an operand other than its two inputs. +fn invalid_binary_operand() -> DataFusionError { + DataFusionError::Internal("invalid binary operand index".to_string()) } -/// The error for an aggregate node whose fold is missing. -fn no_aggregate() -> DataFusionError { - DataFusionError::Execution("the node holds no aggregate".to_string()) -} - -/// The error for a node asked to run a vector operation without one. -fn no_vector() -> DataFusionError { - DataFusionError::Execution("the node holds no vector operation".to_string()) -} - -/// The error for a node asked to broadcast without the broadcast its kind holds. -fn no_broadcast() -> DataFusionError { - DataFusionError::Execution("the node holds no scalar broadcast".to_string()) -} - -/// The error for a node asked to map without the operation its kind holds. -fn no_map() -> DataFusionError { - DataFusionError::Execution("the node holds no map operation".to_string()) -} - -/// The error for a node asked to select without the selection its kind holds. -fn no_instant() -> DataFusionError { - DataFusionError::Execution("the node holds no instant selection".to_string()) +/// A running executor must own its live state. +fn no_live_execution() -> DataFusionError { + DataFusionError::Internal("the running native execution has no state".to_string()) } /// The error for a node asked to evaluate a window without its evaluator. diff --git a/src/promql/src/native/execute/tests.rs b/src/promql/src/native/execute/tests.rs index f43f2bf62e0..fdcf949976b 100644 --- a/src/promql/src/native/execute/tests.rs +++ b/src/promql/src/native/execute/tests.rs @@ -15,10 +15,11 @@ //! What the runtime has to get right: each layer's grid, per-node accounting, rejection and //! release. +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Weak}; use datafusion::error::DataFusionError; -use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; +use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool, MemoryReservation}; use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue}; use datatypes::arrow::array::{Array, Float64Array, UInt32Array}; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; @@ -68,6 +69,57 @@ use crate::vector_match::{ visited_pairs, }; +/// Watches the final release of a reservation without keeping its histogram payload alive. +#[derive(Debug)] +struct DropOrderPool { + inner: GreedyMemoryPool, + watch: std::sync::Mutex)>>, + released: AtomicBool, + retained_on_release: AtomicBool, +} + +impl std::fmt::Display for DropOrderPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "drop-order test pool") + } +} + +impl MemoryPool for DropOrderPool { + fn name(&self) -> &str { + "drop-order test pool" + } + + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.inner.grow(reservation, additional); + } + + fn try_grow( + &self, + reservation: &MemoryReservation, + additional: usize, + ) -> Result<(), DataFusionError> { + self.inner.try_grow(reservation, additional) + } + + fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { + if reservation.size() == 0 { + let watch = self.watch.lock().unwrap(); + if let Some((name, payload)) = watch.as_ref() + && reservation.consumer().name() == name + { + self.released.store(true, Ordering::SeqCst); + self.retained_on_release + .store(payload.upgrade().is_some(), Ordering::SeqCst); + } + } + self.inner.shrink(reservation, shrink); + } + + fn reserved(&self) -> usize { + self.inner.reserved() + } +} + fn grid(start: i64, step: i64, len: u64) -> Arc { Arc::new(EvalGrid::try_new(GridSpec::try_new(start, step, len).unwrap()).unwrap()) } @@ -360,11 +412,7 @@ fn window_and_limitk_collect_only_unified_contributions() { group: Grouping::By(Arc::from([])), }, )); - let RuntimeOp::Vector { - vector: Some(vector), - .. - } = raw_limitk.op - else { + let RuntimeOp::Vector { vector, .. } = raw_limitk.op else { panic!("expected vector runtime node"); }; assert!( @@ -378,11 +426,7 @@ fn window_and_limitk_collect_only_unified_contributions() { }; let fragmented = map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs]); let fragmented_limitk = build(vector_op(fragmented, limitk())); - let RuntimeOp::Vector { - vector: Some(vector), - .. - } = fragmented_limitk.op - else { + let RuntimeOp::Vector { vector, .. } = fragmented_limitk.op else { panic!("expected vector runtime node"); }; assert!( @@ -392,11 +436,7 @@ fn window_and_limitk_collect_only_unified_contributions() { let unified = merge(map(evaluated_input(Arc::clone(&grid)), vec![MathOp::Abs])); let unified_limitk = build(vector_op(unified, limitk())); - let RuntimeOp::Vector { - vector: Some(vector), - .. - } = unified_limitk.op - else { + let RuntimeOp::Vector { vector, .. } = unified_limitk.op else { panic!("expected vector runtime node"); }; assert!( @@ -1439,13 +1479,15 @@ fn node_metrics(execution: &NativeExecutor) -> Vec> { input, parameter, .. } => { walk(input, out); - walk(parameter, out); + if let Some(parameter) = parameter { + walk(parameter, out); + } } RuntimeOp::ParameterizedHistogram { input, parameters, .. } => { walk(input, out); - for parameter in parameters { + for parameter in parameters.iter().flatten() { walk(parameter, out); } } @@ -1460,7 +1502,7 @@ fn node_metrics(execution: &NativeExecutor) -> Vec> { } } let mut out = Vec::new(); - walk(&execution.state().root, &mut out); + walk(&execution.state().live.as_ref().unwrap().root, &mut out); out } @@ -1491,7 +1533,11 @@ fn gauge(set: &ExecutionPlanMetricsSet, name: &str) -> usize { /// it: the trees these tests build have one node of that kind, and it is the root. The state owns /// that tree, so the read is a borrow of it and the execution can be polled again afterwards. fn root_holds(execution: &NativeExecutor, holds: fn(&RuntimeOp) -> bool) -> bool { - holds(&execution.state().root.op) + execution + .state() + .live + .as_ref() + .is_some_and(|live| holds(&live.root.op)) } /// The payload a window root holds: its range evaluator. @@ -1507,24 +1553,12 @@ fn holds_window(op: &RuntimeOp) -> bool { /// The payload an instant root holds: the chunks of the series it is gathering. fn holds_instant(op: &RuntimeOp) -> bool { - matches!( - op, - RuntimeOp::Instant { - instant: Some(_), - .. - } - ) + matches!(op, RuntimeOp::Instant { .. }) } /// The payload a binary root holds: the operands it retained. fn holds_binary(op: &RuntimeOp) -> bool { - matches!( - op, - RuntimeOp::Binary { - binary: Some(_), - .. - } - ) + matches!(op, RuntimeOp::Binary { .. }) } /// Whether the root is the leaf that reads the first routing slot, which is what a tree of one @@ -1533,6 +1567,154 @@ fn holds_input(op: &RuntimeOp) -> bool { matches!(op, RuntimeOp::Input(id) if id.index() == 0) } +/// Both queued input and retained operation payload must die before their reservations return. +#[test] +fn terminal_teardown_drops_payloads_before_returning_reservations() { + for retained in [false, true] { + let watched = Arc::new(DropOrderPool { + inner: GreedyMemoryPool::new(1 << 20), + watch: std::sync::Mutex::new(None), + released: AtomicBool::new(false), + retained_on_release: AtomicBool::new(false), + }); + let pool: Arc = watched.clone(); + let grid = grid(0, 1, 1); + let schema = evaluated_schema(Arc::clone(&grid)); + let root = if retained { + binary_of( + evaluated_input(Arc::clone(&grid)), + evaluated_input(Arc::clone(&grid)), + binary_matching(BinaryOp::Add), + ) + } else { + evaluated_input(Arc::clone(&grid)) + }; + let tree = NativeTree::try_new(root).unwrap(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = metrics_of(&execution); + let table = histogram_table(1, 2); + let weak = Arc::downgrade(&table); + let chunk = evaluated_histogram_chunk_of(&schema, metric(), 0, vec![Some(0)], &table, true); + drop(table); + execution.push_evaluated(InputId::new(0), chunk).unwrap(); + if retained { + assert!(execution.next_chunk().unwrap().is_none()); + } + assert!(weak.upgrade().is_some()); + assert!(pool.reserved() > 0); + let consumer = if retained { + "native.binary at root" + } else { + "native.input at root (source_index=0)" + }; + *watched.watch.lock().unwrap() = Some((consumer.to_string(), weak.clone())); + + // Invalid routing fails while the queue or operation payload is still held. + let invalid = if retained { 2 } else { 1 }; + let error = execution + .push_evaluated( + InputId::new(invalid), + evaluated_chunk(&schema, 0, vec![1.0], true), + ) + .unwrap_err(); + assert!(error.to_string().contains("no input"), "{error}"); + assert!(watched.released.load(Ordering::SeqCst)); + assert!( + !watched.retained_on_release.load(Ordering::SeqCst), + "{consumer}" + ); + assert!(weak.upgrade().is_none()); + assert!(execution.state().live.is_none()); + assert_eq!(pool.reserved(), 0); + assert_eq!(gauge(&metrics, "native.root.current_reserved_bytes"), 0); + assert!(gauge(&metrics, "native.p0.root.peak_reserved_bytes") > 0); + assert!( + execution + .next_chunk() + .unwrap_err() + .to_string() + .contains("stopped after an earlier error") + ); + } +} + +/// A window finish failure is reported before its live execution and reservation are dropped. +#[test] +fn finishing_a_window_propagates_its_error_and_drops_the_live_tree() { + let tree = NativeTree::try_new(window(input(), grid(0, 60_000, 4))).unwrap(); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = metrics_of(&execution); + let nodes = node_metrics(&execution); + let schema = raw_schema(); + execution + .push_raw(InputId::new(0), raw_chunk(&schema, &[0], &[1.0], false)) + .unwrap(); + drop(execution.next_chunk().unwrap()); + assert!(pool.reserved() > 0); + // The source-level check normally catches truncation first; leave the evaluator open so the + // actual terminal finish pass exercises its own error and context propagation. + execution.state.live.as_mut().unwrap().inputs[0].open_source_series = false; + execution.end_of_input().unwrap(); + let error = execution.next_chunk().unwrap_err(); + assert!( + error.to_string().contains("native.window at root"), + "{error}" + ); + assert!( + error.to_string().contains("ended before the last chunk"), + "{error}" + ); + assert!(execution.state().live.is_none()); + assert!(nodes.iter().all(|node| node.upgrade().is_none())); + assert_eq!(pool.reserved(), 0); + assert_eq!(gauge(&metrics, "native.root.current_reserved_bytes"), 0); + assert!(gauge(&metrics, "native.p0.root.peak_reserved_bytes") > 0); + assert!( + execution + .next_chunk() + .unwrap_err() + .to_string() + .contains("stopped after an earlier error") + ); +} + +/// The zero-input scalar operations retain their binding while live, not after completion. +#[test] +fn scalar_operations_drop_their_grid_and_topology_on_finish_or_failure() { + for vector in [false, true] { + for fails in [false, true] { + let grid = grid(0, 1, 2); + let weak = Arc::downgrade(&grid); + let time = eval_time(None, Arc::clone(&grid)); + let tree = NativeTree::try_new(if vector { vector_of(time) } else { time }).unwrap(); + let pool = if fails { bounded_pool(0) } else { pool() }; + let mut execution = executor(&tree, Arc::clone(&pool)); + let nodes = node_metrics(&execution); + drop(tree); + drop(grid); + execution.end_of_input().unwrap(); + if fails { + assert!(execution.next_chunk().is_err()); + } else { + drop(execution.next_chunk().unwrap().unwrap()); + assert!(execution.next_chunk().unwrap().is_none()); + } + assert!(weak.upgrade().is_none()); + assert!(execution.state().live.is_none()); + assert!(nodes.iter().all(|node| node.upgrade().is_none())); + assert_eq!(pool.reserved(), 0); + let error = execution.next_chunk().unwrap_err(); + let expected = if fails { + "stopped after an earlier error" + } else { + "already finished" + }; + assert!(error.to_string().contains(expected), "{error}"); + } + } +} + /// A window over raw input evaluates onto its own output grid, and its values are the evaluator's. /// /// The grid is compared by allocation identity, so a window that evaluated on a copy of its @@ -2049,7 +2231,9 @@ fn a_binary_over_a_broadcast_registers_every_node_it_runs() { // One direct pattern per level, on the nodes the constructor made; the borrow ends before the // first push. { - let RuntimeOp::Binary { left, right, .. } = &execution.state().root.op else { + let RuntimeOp::Binary { left, right, .. } = + &execution.state().live.as_ref().unwrap().root.op + else { panic!("the root is not the binary node the tree built"); }; let RuntimeOp::Broadcast { @@ -2302,7 +2486,7 @@ fn a_fanout_enqueue_failure_releases_every_recipient() { let pool = bounded_pool(first_charge); let mut execution = executor(&tree, Arc::clone(&pool)); let ids: Vec<_> = execution.state().input_ids(0).collect(); - let names = &execution.state().inputs; + let names = &execution.state().live.as_ref().unwrap().inputs; assert_ne!( names[0].state.buffer.reservation.consumer().name(), names[1].state.buffer.reservation.consumer().name() @@ -3047,7 +3231,8 @@ fn a_window_that_merges_its_input_releases_what_it_collected() { // The root is the window that collected the fragments, so its own reservation is what the peak // below has to cover. The borrow ends before the close below. let window_held = { - let RuntimeOp::Window { state, .. } = &execution.state().root.op else { + let RuntimeOp::Window { state, .. } = &execution.state().live.as_ref().unwrap().root.op + else { panic!("the root is not the window the tree built"); }; state.buffer.reservation.size() @@ -3120,7 +3305,9 @@ fn a_merge_flush_reserves_every_queued_output() { // the node handed out under, and the evaluator that evaluated them. The borrow ends before the // next pull. let (queued, queued_bytes, retained, emitted, held) = { - let RuntimeOp::Window { window, state, .. } = &execution.state().root.op else { + let RuntimeOp::Window { window, state, .. } = + &execution.state().live.as_ref().unwrap().root.op + else { panic!("the root is not the window the tree built"); }; let queued_bytes = state.queued.iter().map(|(_, bytes)| *bytes).sum::(); @@ -3209,7 +3396,7 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() { window, merge, state, - } = &execution.state().root.op + } = &execution.state().live.as_ref().unwrap().root.op else { panic!("the root is not the window the tree built"); }; @@ -3241,7 +3428,7 @@ fn dropping_the_execution_releases_its_tree_and_keeps_the_registered_metrics() { merge, state, .. - } = &execution.state().root.op + } = &execution.state().live.as_ref().unwrap().root.op else { panic!("the root is not the window the tree built"); }; @@ -5153,13 +5340,13 @@ fn a_source_charges_its_queue_and_gives_it_back() { assert!(source.pop_source().unwrap().is_none()); } -/// Releasing a node drops its queue and returns the reservation once. +/// Dropping a node drops its queue and returns the reservation once. /// /// A node stopped with chunks still queued holds nothing afterwards: the queue goes, and the bytes it /// was reserved for go back to the pool rather than staying held for as long as the caller keeps the /// stopped executor alive. #[test] -fn releasing_a_node_drops_its_queue() { +fn dropping_a_node_drops_its_queue() { let pool = pool(); let mut source = source_runtime(&pool); let schema = raw_schema(); @@ -5175,14 +5362,12 @@ fn releasing_a_node_drops_its_queue() { } assert!(pool.reserved() > 0); - source.release(); - assert!( - source.state.queued.is_empty(), - "the queue outlived its release" - ); - assert_eq!(source.state.buffer.reservation.size(), 0); + let metrics = ExecutionPlanMetricsSet::new(); + source.state.buffer.metrics.register("source", &metrics, 0); + drop(source); assert_eq!(pool.reserved(), 0, "the reservation was not returned"); - assert!(source.pop_source().unwrap().is_none()); + assert_eq!(gauge(&metrics, "native.source.current_reserved_bytes"), 0); + assert!(gauge(&metrics, "native.p0.source.peak_reserved_bytes") > 0); } /// A chunk whose axis and fields disagree is refused, not read past the end of a field. @@ -6333,14 +6518,14 @@ fn binary_runtime_indices_are_assigned_in_preorder() { let RuntimeOp::Binary { left, - binary: Some(root_binary), + binary: root_binary, .. - } = &execution.root.op + } = &execution.live.as_ref().unwrap().root.op else { panic!("the root is not a binary runtime"); }; let RuntimeOp::Binary { - binary: Some(inner_binary), + binary: inner_binary, .. } = &left.op else { @@ -6396,9 +6581,9 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() { let RuntimeOp::Binary { left, right, - binary: Some(root), + binary: root, .. - } = &execution.root.op + } = &execution.live.as_ref().unwrap().root.op else { panic!("expected root binary"); }; @@ -6419,11 +6604,7 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() { panic!("expected histogram"); }; let binary_at = |node: &RuntimeNode| { - let RuntimeOp::Binary { - binary: Some(binary), - .. - } = &node.op - else { + let RuntimeOp::Binary { binary, .. } = &node.op else { panic!("expected vector binary"); }; assert!(Arc::ptr_eq(binary.evidence.as_ref().unwrap(), &evidence)); @@ -6438,10 +6619,16 @@ fn dynamic_nodes_propagate_binary_indices_and_duplicate_evidence() { assert!(Arc::ptr_eq(root.evidence.as_ref().unwrap(), &evidence)); assert_eq!(root.node_index, 0); assert_eq!(binary_at(topk_input), 1); - assert_eq!(scalar_binary_at(topk_parameter), 2); + assert_eq!(scalar_binary_at(topk_parameter.as_ref().unwrap()), 2); assert_eq!(binary_at(histogram_input), 3); - assert_eq!(scalar_binary_at(&histogram_parameters[0]), 4); - assert_eq!(scalar_binary_at(&histogram_parameters[1]), 5); + assert_eq!( + scalar_binary_at(histogram_parameters[0].as_ref().unwrap()), + 4 + ); + assert_eq!( + scalar_binary_at(histogram_parameters[1].as_ref().unwrap()), + 5 + ); } /// A source of evaluated chunks over `count` float fields. @@ -8852,7 +9039,7 @@ fn queued_set_results_hold_their_labels_until_handed_on() { .unwrap(); let start = |pool: &Arc| { let mut execution = executor(&tree, Arc::clone(pool)); - match &execution.state().root.op { + match &execution.state().live.as_ref().unwrap().root.op { RuntimeOp::Binary { state, .. } => assert_eq!( state.buffer.reservation.consumer().name(), "native.set at root" @@ -10361,13 +10548,12 @@ fn an_aggregate_answers_nothing_until_its_whole_input_ends() { // The root is the aggregate the tree built, and the fold is its own payload: what is read // below is a copy of the flag, not a borrow held across the pulls. let computed = |execution: &NativeExecutor| { - let RuntimeOp::Aggregate { aggregate, .. } = &execution.state().root.op else { + let RuntimeOp::Aggregate { aggregate, .. } = + &execution.state().live.as_ref().unwrap().root.op + else { panic!("the root is not the aggregate the tree built"); }; - aggregate - .as_ref() - .expect("the root holds an aggregate") - .computed + aggregate.computed }; execution @@ -10562,7 +10748,8 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() { // reservation and the backing the queue grew to. The borrow ends inside the call, so the pull // below is taken with nothing held. let queue = |execution: &NativeExecutor| -> (usize, usize, usize, usize) { - let RuntimeOp::Aggregate { state, .. } = &execution.state().root.op else { + let RuntimeOp::Aggregate { state, .. } = &execution.state().live.as_ref().unwrap().root.op + else { panic!("the root is not the aggregate the tree built"); }; ( @@ -10590,10 +10777,11 @@ fn aggregate_runtime_queue_keeps_only_backing_after_pops_until_release() { assert!(execution.next_chunk().unwrap().is_none()); // The terminal pull is the close: the queue's backing and the reservation go back with it, and // the root reads released rather than only uncharged. - let (queued, _, held, capacity) = queue(&execution); - assert_eq!(queued, 0); - assert_eq!(capacity, 0, "the closed root kept the queue's backing"); - assert_eq!(held, 0); + assert!( + execution.state().live.is_none(), + "the closed root was retained" + ); + assert_eq!(execution.state().reserved_bytes(), 0); assert_eq!(pool.reserved(), 0); } @@ -10642,15 +10830,16 @@ fn aggregate_fold_state_is_released_after_input() { // The root is the aggregate the tree built: its fold holds one group and has not run. The // borrow ends before the close below takes the state again. { - let RuntimeOp::Aggregate { aggregate, .. } = &execution.state().root.op else { + let RuntimeOp::Aggregate { aggregate, .. } = + &execution.state().live.as_ref().unwrap().root.op + else { panic!("the root is not the aggregate the tree built"); }; - let aggregate = aggregate.as_ref().expect("the root holds an aggregate"); assert_eq!(aggregate.groups(), 1); assert!(!aggregate.computed); } assert!(pool.reserved() > 0); - execution.state.close().unwrap(); + execution.finish().unwrap(); assert_eq!(pool.reserved(), 0); } @@ -11136,6 +11325,265 @@ fn parameterized_topk_reserves_the_values_of_its_parameter() { } } +/// A consumed dynamic parameter drops charged scalar/broadcast state before the input is finished. +#[test] +fn dynamic_parameters_drop_broadcast_subtrees_before_input_finishes() { + for histogram in [false, true] { + let grid = grid(1_000, 1_000, 2); + let input = evaluated_input(Arc::clone(&grid)); + // Both sides are varying scalars: the broadcast retains one scalar series and emitted + // label history after producing the parameter. + let parameter = scalar_binary( + eval_time(None, Arc::clone(&grid)), + eval_time(None, Arc::clone(&grid)), + binary_matching(BinaryOp::Add), + ); + let root = NativeNode { + output: input.output.clone(), + expr: if histogram { + NativeExpr::ParameterizedHistogram { + input: Box::new(input), + op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)), + parameters: vec![parameter], + } + } else { + NativeExpr::ParameterizedTopK { + input: Box::new(input), + op: BoundVectorOp::TopK { + k: 0.0, + top: true, + group: Grouping::By(Arc::from([])), + }, + parameter: Box::new(parameter), + } + }, + }; + let tree = NativeTree::try_new(root).unwrap(); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = metrics_of(&execution); + let parameter_name = if histogram { + "root.parameter0" + } else { + "root.parameter" + }; + assert!(execution.next_chunk().unwrap().is_none()); + let root = &execution.state().live.as_ref().unwrap().root; + match &root.op { + RuntimeOp::ParameterizedTopK { + parameter, + vector, + state, + .. + } => { + assert!(parameter.is_none()); + assert_eq!( + vector.parameter.as_deref(), + Some([Some(2.0), Some(4.0)].as_slice()) + ); + assert_eq!( + state.buffer.reservation.size(), + 2 * std::mem::size_of::>() + ); + } + RuntimeOp::ParameterizedHistogram { + parameters, + values, + state, + .. + } => { + assert!(parameters[0].is_none()); + assert_eq!( + values[0].as_deref(), + Some([Some(2.0), Some(4.0)].as_slice()) + ); + assert_eq!( + state.buffer.reservation.size(), + 2 * std::mem::size_of::>() + ); + } + _ => panic!("expected dynamic operator"), + } + assert_eq!(execution.state().reserved_bytes(), pool.reserved()); + assert_eq!(pool.reserved(), 2 * std::mem::size_of::>()); + assert_eq!( + gauge( + &metrics, + &format!("native.{parameter_name}.current_reserved_bytes") + ), + 0 + ); + assert!( + gauge( + &metrics, + &format!("native.p0.{parameter_name}.peak_reserved_bytes") + ) > 0 + ); + assert_eq!( + count(&metrics, parameter_name, NodeMetric::OUTPUT_BATCHES), + 1 + ); + assert!(execution.finish().unwrap().is_empty()); + assert_eq!(pool.reserved(), 0); + } +} + +/// `scalar()` releases its own retained input after producing, but a map below it still holds +/// charged emitted-label history. Consuming the parameter must drop that subtree as well. +#[test] +fn dynamic_parameters_drop_scalar_of_map_history_before_terminal_close() { + for histogram in [false, true] { + let grid = grid(0, 1_000, 2); + let schema = evaluated_schema(Arc::clone(&grid)); + let input = evaluated_input(Arc::clone(&grid)); + let parameter = scalar_of(map( + evaluated_input_with_source_index(Arc::clone(&grid), 1), + vec![MathOp::Abs], + )); + let root = NativeNode { + output: input.output.clone(), + expr: if histogram { + NativeExpr::ParameterizedHistogram { + input: Box::new(input), + op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)), + parameters: vec![parameter], + } + } else { + NativeExpr::ParameterizedTopK { + input: Box::new(input), + op: BoundVectorOp::TopK { + k: 0.0, + top: true, + group: Grouping::By(Arc::from([])), + }, + parameter: Box::new(parameter), + } + }, + }; + let tree = NativeTree::try_new(root).unwrap(); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let metrics = metrics_of(&execution); + let parameter_name = if histogram { + "root.parameter0" + } else { + "root.parameter" + }; + let map_name = format!("{parameter_name}.input"); + execution + .push_evaluated( + InputId::new(1), + evaluated_chunk(&schema, 0, vec![1.0, 1.0], true), + ) + .unwrap(); + assert!(execution.next_chunk().unwrap().is_none()); + let map_current = format!("native.{map_name}.current_reserved_bytes"); + assert!( + gauge(&metrics, &map_current) > 0, + "map did not retain emitted history" + ); + // Drive exactly the finishing stage that produces the scalar, before the executor closes. + assert!(execution.live_mut().unwrap().pull(true).unwrap().is_none()); + assert_eq!(gauge(&metrics, &map_current), 0); + assert!( + gauge( + &metrics, + &format!("native.p0.{map_name}.peak_reserved_bytes") + ) > 0 + ); + assert_eq!(count(&metrics, &map_name, NodeMetric::OUTPUT_BATCHES), 1); + let root = &execution.state().live.as_ref().unwrap().root; + match &root.op { + RuntimeOp::ParameterizedTopK { parameter, .. } => assert!(parameter.is_none()), + RuntimeOp::ParameterizedHistogram { parameters, .. } => { + assert!(parameters[0].is_none()) + } + _ => panic!("expected dynamic operator"), + } + assert_eq!(execution.state().reserved_bytes(), pool.reserved()); + execution.live_mut().unwrap().root.finish_windows().unwrap(); + drop(execution); + assert_eq!(pool.reserved(), 0); + } +} + +/// A settled parameter with an open window must report the window's finish error before dropping it. +#[test] +fn dynamic_parameters_finish_windows_before_dropping_the_subtree() { + for histogram in [false, true] { + let grid = grid(0, 1_000, 2); + let input = evaluated_input(Arc::clone(&grid)); + let parameter_window = window(raw_input_of(1), Arc::clone(&grid)); + let parameter = scalar_of(parameter_window); + let root = NativeNode { + output: input.output.clone(), + expr: if histogram { + NativeExpr::ParameterizedHistogram { + input: Box::new(input), + op: BoundVectorOp::Histogram(HistogramFunction::Quantile(0.0)), + parameters: vec![parameter], + } + } else { + NativeExpr::ParameterizedTopK { + input: Box::new(input), + op: BoundVectorOp::TopK { + k: 0.0, + top: true, + group: Grouping::By(Arc::from([])), + }, + parameter: Box::new(parameter), + } + }, + }; + let tree = NativeTree::try_new(root).unwrap(); + let pool = pool(); + let mut execution = executor(&tree, Arc::clone(&pool)); + let live = execution.state.live.as_mut().unwrap(); + let parameter = match &mut live.root.op { + RuntimeOp::ParameterizedTopK { parameter, .. } => parameter.as_deref_mut().unwrap(), + RuntimeOp::ParameterizedHistogram { parameters, .. } => parameters[0].as_mut().unwrap(), + _ => panic!("expected dynamic operator"), + }; + let RuntimeOp::ScalarOf { + input, + produced, + state, + .. + } = &mut parameter.op + else { + panic!("expected scalar() parameter"); + }; + let RuntimeOp::Window { window, .. } = &mut input.op else { + panic!("expected parameter window"); + }; + // Force a queued scalar and an open evaluator, an otherwise unreachable invariant break, + // so the real finish failure cannot be bypassed by ordinary EOF checks. + let chunk = raw_chunk(&raw_schema(), &[0], &[1.0], false); + drop(window.as_mut().unwrap().push(&chunk).unwrap()); + let bytes = grid_chunk_bytes(&grid).unwrap(); + state.buffer.grow(bytes).unwrap(); + state.queued.push_back(( + NativeChunk::Evaluated(super::scalar_chunk(Arc::clone(&grid), 1.0).unwrap()), + bytes, + )); + *produced = true; + execution.inputs_finished = true; + let error = execution.next_chunk().unwrap_err(); + assert!( + error.to_string().contains("ended before the last chunk"), + "{error}" + ); + assert!( + error + .to_string() + .contains("native.window at root.parameter"), + "{error}" + ); + assert!(execution.state().live.is_none()); + assert_eq!(pool.reserved(), 0); + } +} + /// A parameterized `topk` released before it computes gives back everything it holds: the input it /// retained, and with it the samples and the label allocations those chunks keep alive. /// @@ -11191,7 +11639,7 @@ fn a_parameterized_topk_released_before_it_computes_drops_its_retained_input() { pool.reserved() > 0, "the retained chunk was not charged for" ); - execution.state.close().unwrap(); + drop(execution); assert_eq!(pool.reserved(), 0, "the reservation was not returned"); assert_eq!( Arc::strong_count(&job), @@ -11311,8 +11759,7 @@ fn parameterized_histogram_hands_over_its_results_from_its_own_queue() { ("host=b".to_string(), vec![(0, Some(1.5)), (1, Some(1.5))]), ] ); - // The queue is drained, so what is left is the values the parameters were read into. - execution.state.close().unwrap(); + // The terminal pull dropped the queue and parameter values with the live tree. assert_eq!(pool.reserved(), 0); } @@ -11936,7 +12383,7 @@ fn scalar_of_reports_what_it_holds_while_its_input_is_retained() { pool.reserved(), "the reported reservation is not what the node holds" ); - execution.state.close().unwrap(); + drop(execution); assert_eq!(pool.reserved(), 0, "the reservation was not returned"); } @@ -12559,11 +13006,9 @@ fn a_label_change_admits_the_label_it_writes_before_building_it() { assert_eq!(pool.reserved(), 0); } -/// An ended execution keeps its tree and its registered counters, but neither the input schema nor -/// the grid it is addressed on. The leaf itself stays -- the root still says which routing slot it -/// is, and the handles the caller registered before the run still read what the nodes recorded -- -/// but the slot that fed it is taken. Both ways an execution ends are covered: the answer is drained, -/// and a rejected push stops it first. +/// An ended execution drops its tree, input schema and grid, while registered counters survive. +/// Both ways an execution ends are covered: the answer is drained, and a rejected push stops it +/// first. #[test] fn an_ended_execution_keeps_no_input_schema_or_grid() { for fails in [false, true] { @@ -12642,13 +13087,8 @@ fn an_ended_execution_keeps_no_input_schema_or_grid() { assert_eq!(execution.state().reserved_bytes(), 0); assert_eq!(pool.reserved(), 0); assert!( - execution.state().inputs.is_empty(), - "the ended execution kept an input slot" - ); - assert_eq!( - execution.state().inputs.capacity(), - 0, - "the ended execution kept the slot vector's backing" + execution.state().live.is_none(), + "the ended execution kept its input slots" ); let current = format!("native.root.{}", NodeMetric::CURRENT_RESERVED_BYTES); assert_eq!( @@ -12662,8 +13102,8 @@ fn an_ended_execution_keeps_no_input_schema_or_grid() { "the peak the leaf reached did not survive the end of the execution" ); assert!( - root_holds(&execution, holds_input), - "the ended execution replaced the root it was built from" + !root_holds(&execution, holds_input), + "the ended execution kept its root" ); } }