mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 16:24:07 +00:00
fix(promql): keep a shared histogram table charged until the last queued result leaves
Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
This commit is contained in:
@@ -2114,6 +2114,10 @@ struct NodeRuntime {
|
||||
/// uses it for both. A chunk is measured once, as it enters, and the figure it was queued with
|
||||
/// is given back as it leaves, so neither the queue nor the handover walks it.
|
||||
queued: VecDeque<(NativeChunk, usize)>,
|
||||
/// The histogram tables queued results share with each other, charged to the queue as a whole
|
||||
/// rather than to one result: a table stays alive while any result referencing it is queued,
|
||||
/// so its charge is given back when the queue empties, not when the first result leaves.
|
||||
queued_tables: usize,
|
||||
/// What an instant node holds: the chunks of the series it is selecting, and, while it gathers
|
||||
/// that series, the workspace the gather reserved for. `None` for every other kind of node.
|
||||
instant: Option<InstantRuntime>,
|
||||
@@ -2190,6 +2194,7 @@ impl NodeRuntime {
|
||||
// hold it.
|
||||
let _timer = metrics.timer();
|
||||
self.buffer.shrink(bytes)?;
|
||||
self.release_queued_tables()?;
|
||||
}
|
||||
metrics.record_output(1, values(&chunk));
|
||||
// An evaluated chunk is addressed on its grid, so its decided positions are grid points; a
|
||||
@@ -2427,9 +2432,10 @@ impl NodeRuntime {
|
||||
///
|
||||
/// `shares_input` says the series are the assembled input itself, as a sort hands it back: a
|
||||
/// field there is one slice per input chunk of buffers and histogram tables the chunks shared,
|
||||
/// so each series is measured the way the input was ([`operand_chunk_bytes`], each table billed
|
||||
/// once) rather than by the allocations its slices belong to, which would count a buffer once
|
||||
/// per chunk.
|
||||
/// so each series is measured the way the input was ([`operand_chunk_bytes`]) rather than by
|
||||
/// the allocations its slices belong to, which would count a buffer once per chunk. The tables
|
||||
/// are billed once, to the queue ([`NodeRuntime::queued_tables`]), since several results can
|
||||
/// keep one alive.
|
||||
fn queue_results(&mut self, output: Vec<Series>, shares_input: bool) -> Result<()> {
|
||||
let mut tables = HashSet::new();
|
||||
// The results are queued the way a node that produces a chunk per input queues what it
|
||||
@@ -2441,7 +2447,8 @@ impl NodeRuntime {
|
||||
let shared = shares_input.then(|| {
|
||||
let (fresh, table_bytes) = unbilled_tables(&series, &tables);
|
||||
tables.extend(fresh);
|
||||
operand_chunk_bytes(&series).saturating_add(table_bytes)
|
||||
self.queued_tables = self.queued_tables.saturating_add(table_bytes);
|
||||
operand_chunk_bytes(&series)
|
||||
});
|
||||
let chunk = NativeChunk::Evaluated(EvaluatedChunk::try_new(series, true)?);
|
||||
// The result is already admitted, within the workspace reserved for the kernel. What the
|
||||
@@ -2460,7 +2467,9 @@ impl NodeRuntime {
|
||||
let queued = self
|
||||
.queued
|
||||
.iter()
|
||||
.fold(0usize, |total, (_, bytes)| total.saturating_add(*bytes));
|
||||
.fold(self.queued_tables, |total, (_, bytes)| {
|
||||
total.saturating_add(*bytes)
|
||||
});
|
||||
self.buffer.resize(queued)
|
||||
}
|
||||
|
||||
@@ -2622,9 +2631,19 @@ impl NodeRuntime {
|
||||
return Ok(None);
|
||||
};
|
||||
self.buffer.shrink(bytes)?;
|
||||
self.release_queued_tables()?;
|
||||
Ok(Some(chunk))
|
||||
}
|
||||
|
||||
/// Gives back the shared tables' charge once no queued result references them.
|
||||
fn release_queued_tables(&mut self) -> Result<()> {
|
||||
if self.queued.is_empty() && self.queued_tables > 0 {
|
||||
self.buffer
|
||||
.shrink(std::mem::take(&mut self.queued_tables))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Records what this node hands out under its output label set.
|
||||
///
|
||||
/// A node that rewrites labels is where two input series become indistinguishable, so this is
|
||||
@@ -2666,6 +2685,7 @@ impl NodeRuntime {
|
||||
self.open_source_series = false;
|
||||
self.queued.clear();
|
||||
self.queued.shrink_to_fit();
|
||||
self.queued_tables = 0;
|
||||
// The reservation is returned once, below, for everything this node still held.
|
||||
self.buffer.release();
|
||||
}
|
||||
@@ -2757,6 +2777,7 @@ impl NativeExecutionState {
|
||||
let mut runtime = NodeRuntime {
|
||||
buffer,
|
||||
queued: VecDeque::new(),
|
||||
queued_tables: 0,
|
||||
instant: None,
|
||||
window: None,
|
||||
map: None,
|
||||
@@ -2963,10 +2984,11 @@ impl NativeExecutionState {
|
||||
_ => None,
|
||||
};
|
||||
let limitk = matches!(op, Some(BoundVectorOp::LimitK { .. }));
|
||||
if limitk && input_holds_shared {
|
||||
if let Some(vector) = runtimes[index].vector.as_mut() {
|
||||
vector.merge_input = true;
|
||||
}
|
||||
if limitk
|
||||
&& input_holds_shared
|
||||
&& let Some(vector) = runtimes[index].vector.as_mut()
|
||||
{
|
||||
vector.merge_input = true;
|
||||
}
|
||||
let whole = limitk || matches!(op, Some(BoundVectorOp::Absent { .. }));
|
||||
holds_shared[index] = input_holds_shared && !whole;
|
||||
|
||||
@@ -479,6 +479,7 @@ fn source_runtime(pool: &Arc<dyn MemoryPool>) -> NodeRuntime {
|
||||
pool,
|
||||
),
|
||||
queued: VecDeque::new(),
|
||||
queued_tables: 0,
|
||||
instant: None,
|
||||
window: None,
|
||||
map: None,
|
||||
@@ -496,6 +497,7 @@ fn binary_runtime(pool: &Arc<dyn MemoryPool>, schema: &Arc<SeriesSchema>) -> Nod
|
||||
NodeRuntime {
|
||||
buffer: NodeBuffer::new(&NodePath::root(), Arc::new(NodeMetrics::default()), pool),
|
||||
queued: VecDeque::new(),
|
||||
queued_tables: 0,
|
||||
instant: None,
|
||||
window: None,
|
||||
map: None,
|
||||
@@ -6705,3 +6707,75 @@ fn a_sorted_series_is_queued_at_what_its_slices_hold() {
|
||||
assert_eq!(decided(&chunks).len(), points as usize);
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// Two sorted results share one histogram table, so the table stays charged until the second one
|
||||
/// leaves, not only until the first one does.
|
||||
#[test]
|
||||
fn a_table_shared_by_queued_results_stays_charged_until_the_last_leaves() {
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(vector_op(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
BoundVectorOp::Sort(SortOrder::ByLabel(vec!["job".into()])),
|
||||
))
|
||||
.unwrap();
|
||||
let table = histogram_table(2, 64);
|
||||
let pool = pool();
|
||||
let mut execution = executor(&tree, Arc::clone(&pool), &ExecutionPlanMetricsSet::new());
|
||||
for (job, index) in [("a", 0), ("b", 1)] {
|
||||
execution
|
||||
.push_evaluated(evaluated_histogram_chunk_of(
|
||||
&schema,
|
||||
named_metric("m", job),
|
||||
0,
|
||||
vec![Some(index)],
|
||||
&table,
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
execution.end_of_input().unwrap();
|
||||
let first = execution.next_chunk().unwrap().expect("two sorted results");
|
||||
assert_eq!(first.series.metric.get("job"), Some("a"));
|
||||
drop(first);
|
||||
assert!(pool.reserved() >= histogram_table_bytes(&table));
|
||||
let second = execution.next_chunk().unwrap().expect("two sorted results");
|
||||
assert_eq!(second.series.metric.get("job"), Some("b"));
|
||||
assert!(execution.next_chunk().unwrap().is_none());
|
||||
assert_eq!(pool.reserved(), 0);
|
||||
}
|
||||
|
||||
/// `limitk` takes the order a sort below it decided: `sort_desc` puts `{job="z"}` first, so it is
|
||||
/// the one `limitk(1, ...)` keeps, although `{job="a"}` sorts first by label.
|
||||
#[test]
|
||||
fn limitk_keeps_the_order_a_sort_below_it_decided() {
|
||||
let grid = grid(0, 60_000, 1);
|
||||
let schema = evaluated_schema(Arc::clone(&grid));
|
||||
let tree = NativeTree::try_new(vector_op(
|
||||
vector_op(
|
||||
evaluated_input(Arc::clone(&grid)),
|
||||
BoundVectorOp::Sort(SortOrder::Desc),
|
||||
),
|
||||
BoundVectorOp::LimitK {
|
||||
k: 1.0,
|
||||
group: Grouping::By(Arc::from([])),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
for (job, value) in [("a", 1.0), ("z", 2.0)] {
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric("v", job),
|
||||
0,
|
||||
vec![Some(value)],
|
||||
true,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
decided_by_job(&run(&mut execution)),
|
||||
vec![("z".to_string(), vec![(0, Some(2.0))])]
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user