mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-22 02:34:00 +00:00
fix(promql): refuse an interleaved input, and close a label set only for its owner
Two series-level identity holes, both reported against `42ea525`.
The truncation flag followed the last chunk handed over, which is the same as
"is a series open" only while at most one series can be open. A tree with no
window still accepted `Interleaved`, and `a` open, `b` closed, end of input then
passed while `a` never closed. Tracking the open set would need an identity that
a raw chunk in bound-label mode does not have (`metric` is `None`, the key holds
it), so the ordering is refused when the state is built instead, for every tree
shape and not only under a window. The flag's doc now says what it means and why
that is the whole answer.
`History` closed on any series' `is_last`, so with `abs` dropping the metric name
a series that contributed nothing could close a label set another series was
still filling: `a{job="x"}` value, `b{job="x"}` empty close, `a{job="x"}`
continuation was refused with "already closed the label set". Contiguous input
hands over exactly that sequence, so it was not an interleaving artefact. A
history now remembers the series that filled it and closes only for that series'
closing chunk; a series that produced nothing under an open set is passed over,
as it already was under a closed one. The owner is charged as part of the entry.
The ordering check also reads every input node rather than only the first: a tree
with two inputs would otherwise have its second declaration read and dropped.
Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
@@ -51,13 +51,13 @@ use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReser
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datatypes::arrow::array::{Array, Float64Array};
|
||||
use datatypes::arrow::buffer::Buffer;
|
||||
use datatypes::series::SeriesOrdering;
|
||||
use datatypes::series::histogram::HistogramBatch;
|
||||
use datatypes::series::raw::RawSeriesChunk;
|
||||
use datatypes::series::samples::{
|
||||
DenseSamplesRef, SampleRef, Samples, Series, SeriesFields, SeriesSchema,
|
||||
};
|
||||
use datatypes::series::time::{GridRange, TimeAxis};
|
||||
use datatypes::series::{MetricName, SeriesOrdering};
|
||||
|
||||
use crate::grid_series::{LabelSet, label_set};
|
||||
use crate::map::{MathOp, drop_metric_name, map_math};
|
||||
@@ -99,6 +99,17 @@ impl NativeChunk {
|
||||
}
|
||||
}
|
||||
|
||||
/// The identity of the series this chunk belongs to, when the chunk names it.
|
||||
///
|
||||
/// A raw chunk in bound-label mode carries its identity in its key and no metric name, so there
|
||||
/// is nothing to compare; an evaluated chunk always names the series it holds.
|
||||
fn metric(&self) -> Option<&MetricName> {
|
||||
match self {
|
||||
Self::Raw(chunk) => chunk.metric.as_ref(),
|
||||
Self::Evaluated(chunk) => Some(&chunk.series.metric),
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes this chunk holds, counting each shared allocation it references that `charged` has
|
||||
/// not met yet.
|
||||
fn bytes(&self, charged: &mut ChargedAllocations) -> usize {
|
||||
@@ -528,13 +539,37 @@ struct Emitted {
|
||||
struct History {
|
||||
/// The occupied runs of each field, in position order and disjoint.
|
||||
fields: Vec<Vec<Run>>,
|
||||
/// The source series that filled this label set, when the chunk that did named it.
|
||||
///
|
||||
/// A chunk holding no value is handed over to close a series, never to fill a label set, so it
|
||||
/// closes this one only when it belongs to the series that filled it: a second series that
|
||||
/// produced nothing here is a series this label set never saw, and closing on its `is_last`
|
||||
/// would leave the series that did produce the values unable to finish. `None` is a set filled
|
||||
/// by chunks that named no series, where there is nothing to compare a closing chunk against.
|
||||
owner: Option<MetricName>,
|
||||
/// Whether the series on this label set closed. A second series cannot close it again.
|
||||
closed: bool,
|
||||
/// The heap this entry owns: the runs its per-field vectors hold, kept current by
|
||||
/// [`Emitted::occupy`]. The header and the slot it sits in are charged by the table's capacity.
|
||||
/// The heap this entry owns: the runs its per-field vectors hold and the owner's name and tags,
|
||||
/// kept current by [`Emitted::occupy`]. The header and the slot it sits in are charged by the
|
||||
/// table's capacity.
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl History {
|
||||
/// Whether a chunk of `source` holding no value is the one that closes this label set.
|
||||
///
|
||||
/// Only the series that filled it can: a series that produced nothing here is one this label
|
||||
/// set never saw, so closing on its `is_last` would take the close away from the series that
|
||||
/// did fill it. A chunk naming no series is compared with nothing and closes the set, which is
|
||||
/// what every chunk did before the owner was kept.
|
||||
fn closed_by(&self, source: Option<&MetricName>) -> bool {
|
||||
match (self.owner.as_ref(), source) {
|
||||
(Some(owner), Some(source)) => owner == source,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes a label set holds: each label's name and value with the pair that refers to them.
|
||||
fn label_bytes(labels: &LabelSet) -> usize {
|
||||
labels
|
||||
@@ -543,7 +578,23 @@ fn label_bytes(labels: &LabelSet) -> usize {
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// The bytes a history holds: the per-field vectors and the runs in them.
|
||||
/// The bytes a metric name holds: its name and each tag's name and value, with the tags that refer
|
||||
/// to them.
|
||||
///
|
||||
/// This is the convention [`label_bytes`] keeps: the payload of the shared strings and one row per
|
||||
/// label. The `Option<MetricName>` a [`History`] keeps its owner in is not counted here, being part
|
||||
/// of the header that entry's slot is charged for.
|
||||
fn metric_bytes(metric: &MetricName) -> usize {
|
||||
metric.name().map_or(0, |name| name.len())
|
||||
+ std::mem::size_of_val(metric.tags())
|
||||
+ metric
|
||||
.tags()
|
||||
.iter()
|
||||
.map(|tag| tag.name.len() + tag.value.len())
|
||||
.sum::<usize>()
|
||||
}
|
||||
|
||||
/// The bytes a history holds: the per-field vectors, the runs in them and its owner.
|
||||
///
|
||||
/// This is what the entry itself allocates, not the slot it is filed under: the `History` lives in
|
||||
/// the map's slot, and that slot is what [`Emitted::bytes`] charges with the table's capacity. A
|
||||
@@ -556,11 +607,16 @@ fn history_bytes(history: &History) -> usize {
|
||||
.iter()
|
||||
.map(|runs| runs.capacity() * std::mem::size_of::<Run>())
|
||||
.sum::<usize>()
|
||||
+ history.owner.as_ref().map_or(0, metric_bytes)
|
||||
}
|
||||
|
||||
impl Emitted {
|
||||
/// Records what `chunk` occupies, rejecting it against what this node already handed out.
|
||||
///
|
||||
/// `source` is the identity of the input series `chunk` came from, when the chunk names one: a
|
||||
/// label set is filled by the series that contributes its values, and it is that series whose
|
||||
/// closing chunk closes it (see [`History::owner`]).
|
||||
///
|
||||
/// A label set that has closed cannot be reached by a second series: the node would hand on two
|
||||
/// chunks that its consumer cannot tell apart, and the merge the caller wants spans the first
|
||||
/// series, whose state the evaluator above has already closed. Such a pair does merge in
|
||||
@@ -568,7 +624,12 @@ impl Emitted {
|
||||
/// refused instead of being answered wrongly (see [`closed_label_set`]). A chunk of a set still
|
||||
/// open is refused only where it holds a value at a position that same field already holds one
|
||||
/// at, which is the case no merge can resolve.
|
||||
fn occupy(&mut self, path: &NodePath, chunk: &EvaluatedChunk) -> Result<()> {
|
||||
fn occupy(
|
||||
&mut self,
|
||||
path: &NodePath,
|
||||
source: Option<&MetricName>,
|
||||
chunk: &EvaluatedChunk,
|
||||
) -> Result<()> {
|
||||
let labels = label_set(&chunk.series.metric);
|
||||
let occupied: Vec<Vec<Run>> = chunk
|
||||
.series
|
||||
@@ -577,18 +638,18 @@ impl Emitted {
|
||||
.enumerate()
|
||||
.map(|(index, field)| occupied_runs(path, index, field, &chunk.series.time))
|
||||
.collect::<Result<_>>()?;
|
||||
// A chunk holding no value records nothing. If the label set is unknown or already closed,
|
||||
// the series this chunk belongs to contributed no sample at all -- either it is the first
|
||||
// such series, or a later one that also answers empty -- so there is nothing to report,
|
||||
// nothing to merge and nothing to refuse. A label set still open is the other case: the
|
||||
// chunk closes the series that *did* produce the values, so it falls through to close it.
|
||||
let empty = occupied.iter().all(Vec::is_empty);
|
||||
if empty && self.sets.get(&labels).is_none_or(|history| history.closed) {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(history) = self.sets.get(&labels) {
|
||||
let recorded = self.sets.get(&labels);
|
||||
if let Some(history) = recorded {
|
||||
if history.closed {
|
||||
return Err(closed_label_set(path, &labels));
|
||||
// A series that produced nothing under a closed label set contributes no sample, so
|
||||
// there is nothing to report, nothing to merge and nothing to refuse. One that did
|
||||
// produce values is the second series this set cannot take.
|
||||
return if empty {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(closed_label_set(path, &labels))
|
||||
};
|
||||
}
|
||||
for (index, runs) in occupied.iter().enumerate() {
|
||||
let stored = history.fields.get(index).map_or(&[][..], Vec::as_slice);
|
||||
@@ -604,6 +665,17 @@ impl Emitted {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A chunk holding no value records nothing. Past a closed set the same series is the
|
||||
// one that answered empty, and on an open set it is the close of the series that filled
|
||||
// it -- but only of that series: a different one closes a label set it never wrote to,
|
||||
// which would leave the series that did write to it unable to finish.
|
||||
if empty && !(chunk.is_last && history.closed_by(source)) {
|
||||
return Ok(());
|
||||
}
|
||||
} else if empty {
|
||||
// The first chunk onto a label set holds no value: a series that answered empty, with
|
||||
// nothing to record, nothing to close and nothing to merge.
|
||||
return Ok(());
|
||||
}
|
||||
let fields = chunk.series.fields.len();
|
||||
// The change is folded in here rather than recomputed when the figure is read: a node
|
||||
@@ -617,6 +689,14 @@ impl Emitted {
|
||||
}
|
||||
};
|
||||
let before = history.bytes;
|
||||
// The series that contributes the values is the one that owns the label set, so a chunk of
|
||||
// another one cannot close it later on.
|
||||
if !empty
|
||||
&& let Some(source) = source
|
||||
&& history.owner.as_ref() != Some(source)
|
||||
{
|
||||
history.owner = Some(source.clone());
|
||||
}
|
||||
if history.fields.len() < fields {
|
||||
history.fields.resize_with(fields, Vec::new);
|
||||
}
|
||||
@@ -638,10 +718,11 @@ impl Emitted {
|
||||
/// An estimate of the bytes this table holds, which the node reserves for.
|
||||
///
|
||||
/// It counts each label's name and value bytes with a tuple per label, the runs a history keeps
|
||||
/// in its per-field vectors, and the table's capacity at one label-set slot plus one history
|
||||
/// slot per entry -- which is where a `History` header itself is charged, once, for the entry
|
||||
/// that lives in it. It leaves out the map's buckets and the `Arc` headers of the strings it
|
||||
/// shares, so it is a lower bound rather than a measurement.
|
||||
/// in its per-field vectors, the owner each entry keeps with its name and tags, and the table's
|
||||
/// capacity at one label-set slot plus one history slot per entry -- which is where a `History`
|
||||
/// header itself is charged, once, for the entry that lives in it. It leaves out the map's
|
||||
/// buckets and the `Arc` headers of the strings it shares, so it is a lower bound rather than a
|
||||
/// measurement.
|
||||
///
|
||||
/// The per-entry part is a running total, so reading this after every output does not walk the
|
||||
/// table: only the entry that changed and the table's own capacity are read here.
|
||||
@@ -950,12 +1031,10 @@ struct NodeRuntime {
|
||||
/// consumer saw did not close its series, and no chunk came after it. Nothing else writes it, so
|
||||
/// it cannot report a truncation that did not happen.
|
||||
///
|
||||
/// One flag is enough under the `Contiguous` and `KeyThenTime` orderings, which keep the chunks
|
||||
/// of a series together: only the series last handed over can be open, and one that closed
|
||||
/// cannot resume. An `Interleaved` input does reach a tree with no window above it -- that
|
||||
/// ordering is refused only where a window reads a series at a time -- and there an earlier
|
||||
/// series left open behind another one's chunk is not reported. The flag under-reports such a
|
||||
/// stream; it never over-reports one.
|
||||
/// "The series of the last chunk handed over" is the whole answer because no more than one
|
||||
/// series can be open here: [`NativeExecutionState::try_new`] refuses an input declaring
|
||||
/// `SeriesOrdering::Interleaved`, so the chunks of a series arrive together and the series last
|
||||
/// handed over is the only one that could still be open.
|
||||
open_source_series: bool,
|
||||
}
|
||||
|
||||
@@ -1060,8 +1139,10 @@ impl NodeRuntime {
|
||||
if let Some(output) = &output {
|
||||
metrics.record_grid_points(output.series.time.len());
|
||||
metrics.record_output(1, valid_values(output));
|
||||
// What it handed out is what the label sets of this node now hold.
|
||||
self.track(path, output)?;
|
||||
// What it handed out is what the label sets of this node now hold. The identity is the
|
||||
// input's: `map_chunk` has already rewritten the output's label set, and a series is
|
||||
// the one that arrived.
|
||||
self.track(path, Some(&chunk.series.metric), output)?;
|
||||
}
|
||||
let bytes = self.emitted_bytes();
|
||||
self.buffer.hold_exactly(bytes)?;
|
||||
@@ -1075,6 +1156,12 @@ impl NodeRuntime {
|
||||
chunk.positions(),
|
||||
chunk.is_last(),
|
||||
);
|
||||
// Which series this chunk came from, for the label-set table below. A window consumes one
|
||||
// series to its `is_last` before the next, so the chunk that closes an output label set is
|
||||
// the one of the series that produced its values, which is what lets the table accept a
|
||||
// closing chunk of a series that wrote to it. A raw chunk in bound-label mode names no
|
||||
// series, so there is nothing to pass on.
|
||||
let source = chunk.metric();
|
||||
// The chunk is held for as long as the evaluator consumes it.
|
||||
self.buffer.hold(bytes)?;
|
||||
let metrics = Arc::clone(&self.buffer.metrics);
|
||||
@@ -1098,8 +1185,9 @@ impl NodeRuntime {
|
||||
if let Some(output) = &output {
|
||||
metrics.record_grid_points(output.series.time.len());
|
||||
metrics.record_output(1, valid_values(output));
|
||||
// What it handed out is what the label sets of this node now hold.
|
||||
self.track(path, output)?;
|
||||
// What it handed out is what the label sets of this node now hold, and the series that
|
||||
// filled one is the series its chunks came from.
|
||||
self.track(path, source, output)?;
|
||||
}
|
||||
// The chunk has been consumed: what this node still holds is what the evaluator kept of it
|
||||
// and the label sets it has already handed out. Between pushes the charge for both is the
|
||||
@@ -1131,13 +1219,20 @@ impl NodeRuntime {
|
||||
|
||||
/// Records what this node hands out under its output label set.
|
||||
///
|
||||
/// A node that rewrites labels is where two input series become indistinguishable, so this is
|
||||
/// where the second one is refused: the node above reads what this one hands over, and a window
|
||||
/// above would evaluate the two as two series and report both. Checking at the root would be
|
||||
/// too late for that — the answer would already have been computed.
|
||||
fn track(&mut self, path: &NodePath, output: &EvaluatedChunk) -> Result<()> {
|
||||
/// `source` is the identity of the input series the chunk came from, when there is one: a node
|
||||
/// that rewrites labels is where two input series become indistinguishable, so this is where
|
||||
/// the second one is refused, and it is also the only place that knows which series filled the
|
||||
/// label set a chunk closes. A node above reads what this one hands over, and a window above
|
||||
/// would evaluate the two as two series and report both. Checking at the root would be too late
|
||||
/// for that — the answer would already have been computed.
|
||||
fn track(
|
||||
&mut self,
|
||||
path: &NodePath,
|
||||
source: Option<&MetricName>,
|
||||
output: &EvaluatedChunk,
|
||||
) -> Result<()> {
|
||||
match self.emitted.as_mut() {
|
||||
Some(emitted) => emitted.occupy(path, output),
|
||||
Some(emitted) => emitted.occupy(path, source, output),
|
||||
// A node that keeps its input's labels cannot make two series one.
|
||||
None => Ok(()),
|
||||
}
|
||||
@@ -1209,23 +1304,23 @@ impl NativeExecutionState {
|
||||
// cannot be shown at one position and registered at another.
|
||||
let mut pending: Vec<PendingNode> = Vec::new();
|
||||
let mut unsupported: Option<DataFusionError> = None;
|
||||
// The order the source streams its series in, and where it says so: a window evaluates one
|
||||
// series at a time, so it is only executable over an input that keeps each series
|
||||
// contiguous.
|
||||
let mut ordering: Option<(SeriesOrdering, NodePath)> = None;
|
||||
let mut first_window: Option<NodePath> = None;
|
||||
// The first input whose declared order this runtime cannot execute, and where it says so.
|
||||
// Every node here reads one series at a time, so an input that interleaves them is one this
|
||||
// runtime cannot execute. Recorded per input rather than kept from the first one seen: a
|
||||
// tree with two inputs would otherwise have its second declaration read and dropped.
|
||||
let mut unsupported_ordering_found: Option<(SeriesOrdering, NodePath)> = None;
|
||||
metrics::walk(tree.root(), &mut |node, path| {
|
||||
let handles = Arc::new(NodeMetrics::default());
|
||||
handles.register(path, metrics_set, partition);
|
||||
let plan = match &node.expr {
|
||||
NativeExpr::Input(bound) => {
|
||||
ordering.get_or_insert_with(|| (bound.ordering, path.clone()));
|
||||
if bound.ordering == SeriesOrdering::Interleaved {
|
||||
unsupported_ordering_found
|
||||
.get_or_insert_with(|| (bound.ordering, path.clone()));
|
||||
}
|
||||
Some((NodeTag::Input, Seed::Input))
|
||||
}
|
||||
NativeExpr::Window { op, .. } => {
|
||||
first_window.get_or_insert_with(|| path.clone());
|
||||
Some((NodeTag::Window, Seed::Window(op.clone())))
|
||||
}
|
||||
NativeExpr::Window { op, .. } => Some((NodeTag::Window, Seed::Window(op.clone()))),
|
||||
NativeExpr::Map { op, .. } => Some((NodeTag::Map, Seed::Map(Arc::clone(&op.ops)))),
|
||||
// The kinds below are legal in a bound tree, and `validate` stays the authority on
|
||||
// that; this runtime simply does not execute them yet.
|
||||
@@ -1262,19 +1357,16 @@ impl NativeExecutionState {
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
// A window feeds one series at a time to its evaluator, so it can only run over an input
|
||||
// that keeps each series contiguous. The tree stays free to say `Interleaved` -- the
|
||||
// compiler's job is not narrowed here -- but this runtime says so up front instead of
|
||||
// failing on the second series of a chunk stream it advertised a chunk at a time.
|
||||
if let Some(window) = &first_window
|
||||
&& let Some((ordering, source)) = &ordering
|
||||
{
|
||||
match ordering {
|
||||
SeriesOrdering::Contiguous | SeriesOrdering::KeyThenTime => {}
|
||||
SeriesOrdering::Interleaved => {
|
||||
return Err(unsupported_ordering(*ordering, window, source));
|
||||
}
|
||||
}
|
||||
// Every node here reads one series at a time and assumes its chunks arrive together: a
|
||||
// window feeds a series to its evaluator up to the chunk carrying `is_last`, and an input
|
||||
// node keeps one flag for the series it handed over, which is what reports one that never
|
||||
// closed. An input that declares `Interleaved` can leave a series open across another
|
||||
// series' chunks, which either of those would misread, so the tree is refused here rather
|
||||
// than run. That is a precondition of the runtime, not a limitation to document: the tree
|
||||
// stays free to declare the ordering -- the compiler's job is not narrowed here -- and no
|
||||
// tree this runtime executes declares it.
|
||||
if let Some((ordering, source)) = &unsupported_ordering_found {
|
||||
return Err(unsupported_ordering(*ordering, source));
|
||||
}
|
||||
|
||||
// A node's input is the node at the same position extended by the step the traversal took,
|
||||
@@ -1804,19 +1896,15 @@ fn closed(path: &NodePath) -> DataFusionError {
|
||||
DataFusionError::Execution(format!("the window at {path} is already closed"))
|
||||
}
|
||||
|
||||
/// The rejection of an input ordering this runtime cannot execute under a window.
|
||||
/// The rejection of an input ordering this runtime cannot execute.
|
||||
///
|
||||
/// It names the window that needs contiguity, the input that does not keep it, and the ordering
|
||||
/// that was found, so a reader can tell a tree this runtime cannot run from one it cannot run
|
||||
/// correctly.
|
||||
fn unsupported_ordering(
|
||||
ordering: SeriesOrdering,
|
||||
window: &NodePath,
|
||||
input: &NodePath,
|
||||
) -> DataFusionError {
|
||||
/// It names the input that declares the ordering and the ordering found, so a reader can tell a
|
||||
/// tree this runtime cannot run from one it cannot run correctly.
|
||||
fn unsupported_ordering(ordering: SeriesOrdering, input: &NodePath) -> DataFusionError {
|
||||
DataFusionError::NotImplemented(format!(
|
||||
"the native runtime does not execute windows over {ordering:?} input yet: the window at \
|
||||
{window} reads one series at a time, but the input at {input} is {ordering:?}"
|
||||
"the native runtime does not execute {ordering:?} input yet: the nodes here read one series \
|
||||
at a time, and the input at {input} is {ordering:?}, so the chunks of two of them can \
|
||||
arrive interleaved"
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -1349,9 +1349,13 @@ fn a_merge_across_a_closed_series_is_not_implemented() {
|
||||
/// Two open chunks that share a label set are accepted where no position of a field collides.
|
||||
///
|
||||
/// `a{job="x"}` and `b{job="x"}` are two series to the input and one label set to `abs`, which
|
||||
/// drops the name. The source is interleaved, so the second one is pushed while the first is still
|
||||
/// open, and neither says it is the last chunk of its series. The two occupy disjoint positions of
|
||||
/// field 0 — a value at 0 and a value at 3 — so the label-set check hands both on.
|
||||
/// drops the name. Neither piece says it is the last chunk of its series, so the second one arrives
|
||||
/// while the first is still open. The two occupy disjoint positions of field 0 — a value at 0 and a
|
||||
/// value at 3 — so the label-set check hands both on.
|
||||
///
|
||||
/// The ordering is contiguous because that is the only ordering this runtime executes: the stream
|
||||
/// below is not one a scan produces, since it never closes `a`, and what is checked here is the
|
||||
/// occupancy the check reads rather than a delivery order.
|
||||
///
|
||||
/// This is acceptance by that check, and it is not a merge: this layer passes every chunk through
|
||||
/// as it arrives and keeps no sample state that would combine two series. Merging two series under
|
||||
@@ -1361,10 +1365,10 @@ fn a_merge_across_a_closed_series_is_not_implemented() {
|
||||
fn two_open_series_that_share_a_label_set_are_accepted_when_no_position_collides() {
|
||||
let map_grid = grid(0, 60_000, 4);
|
||||
let tree = NativeTree::try_new(map(
|
||||
evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Interleaved),
|
||||
evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Contiguous),
|
||||
vec![MathOp::Abs],
|
||||
))
|
||||
.expect("a map-only tree is legal over an interleaved input");
|
||||
.expect("a map-only tree is legal over a contiguous input");
|
||||
let schema = evaluated_schema(Arc::clone(&map_grid));
|
||||
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
@@ -1645,6 +1649,12 @@ fn a_chunk_with_no_output_keeps_charging_the_emitted_table() {
|
||||
}
|
||||
|
||||
/// A window cannot run over an interleaved input, and the runtime says so before reading a chunk.
|
||||
///
|
||||
/// The binding refuses a window whose own input declares the ordering, and the runtime refuses the
|
||||
/// ordering itself -- which is what catches a window over a map over the source, where the binding
|
||||
/// only sees the map. A tree with no window above the source is refused by the same check, since
|
||||
/// every node here reads one series at a time; [`an_interleaved_input_is_refused_without_a_window`]
|
||||
/// is that case on its own.
|
||||
#[test]
|
||||
fn a_window_over_an_interleaved_input_is_rejected() {
|
||||
let inner_grid = grid(0, 60_000, 2);
|
||||
@@ -1688,44 +1698,37 @@ fn a_window_over_an_interleaved_input_is_rejected() {
|
||||
);
|
||||
let message = error.to_string();
|
||||
assert!(message.contains("Interleaved"), "{message}");
|
||||
assert!(message.contains("at root "), "{message}");
|
||||
assert!(message.contains("root.input"), "{message}");
|
||||
}
|
||||
|
||||
// A map needs no ordering: it holds one chunk at a time, so the series of an interleaved stream
|
||||
// never have to be contiguous. `A1, B1, A2, B2` runs here as it arrives.
|
||||
let map_grid = grid(0, 60_000, 4);
|
||||
/// An interleaved input is refused with no window above it to notice it.
|
||||
///
|
||||
/// A map over an input has no evaluator, so nothing in the tree reads one series to its `is_last`.
|
||||
/// The sequence the input node cannot tell from a whole one is `a{job="x"}` open, `b{job="y"}`
|
||||
/// closed, and the end of the input: the flag that reports a series left open would say the last
|
||||
/// chunk closed its series and `finish` would accept an input whose first series never closed. The
|
||||
/// runtime therefore refuses the ordering when the state is built, before a chunk is read, instead
|
||||
/// of reporting a truncation it cannot see.
|
||||
#[test]
|
||||
fn an_interleaved_input_is_refused_without_a_window() {
|
||||
let map_grid = grid(0, 60_000, 2);
|
||||
let tree = NativeTree::try_new(map(
|
||||
evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Interleaved),
|
||||
vec![MathOp::Abs],
|
||||
))
|
||||
.expect("a map-only tree is legal over an interleaved input");
|
||||
let schema = evaluated_schema(Arc::clone(&map_grid));
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
for (name, job, start, values, is_last) in [
|
||||
("a", "x", 0, vec![Some(-1.0), Some(-2.0)], false),
|
||||
("b", "y", 0, vec![Some(-3.0), Some(-4.0)], false),
|
||||
("a", "x", 2, vec![Some(-5.0), Some(-6.0)], true),
|
||||
("b", "y", 2, vec![Some(-7.0), Some(-8.0)], true),
|
||||
] {
|
||||
execution
|
||||
.push_evaluated(evaluated_chunk_of(
|
||||
&schema,
|
||||
named_metric(name, job),
|
||||
start,
|
||||
values,
|
||||
is_last,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
addressed(&run(&mut execution)),
|
||||
vec![
|
||||
(0, vec![Some(1.0), Some(2.0)]),
|
||||
(0, vec![Some(3.0), Some(4.0)]),
|
||||
(2, vec![Some(5.0), Some(6.0)]),
|
||||
(2, vec![Some(7.0), Some(8.0)]),
|
||||
]
|
||||
.expect("the binding accepts a map over an interleaved input");
|
||||
let error =
|
||||
match NativeExecutionState::try_new(&tree, pool(), &ExecutionPlanMetricsSet::new(), 0) {
|
||||
Ok(_) => panic!("the runtime accepted an interleaved input with no window above it"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(
|
||||
matches!(error, DataFusionError::NotImplemented(_)),
|
||||
"{error}"
|
||||
);
|
||||
let message = error.to_string();
|
||||
assert!(message.contains("Interleaved"), "{message}");
|
||||
assert!(message.contains("root.input"), "{message}");
|
||||
}
|
||||
|
||||
/// The charge for the label sets a node has handed out does not depend on how it is read.
|
||||
@@ -1781,6 +1784,11 @@ fn the_emitted_charge_is_the_same_folded_in_as_walked() {
|
||||
/// This is the other order of [`two_series_with_no_result_do_not_collide`], and the one that used to
|
||||
/// fail: by the time the empty series arrives the label set is already closed, and refusing it
|
||||
/// failed an execution whose answer was never in doubt.
|
||||
///
|
||||
/// The set is closed here, which is what makes the empty series a later one that also answers
|
||||
/// empty. Its mirror image -- an empty closing chunk that arrives while the set is still open -- is
|
||||
/// [`an_empty_closing_chunk_of_another_series_does_not_close_a_label_set`], where the same series
|
||||
/// is not a harmless extra but would take the close away from the series that filled the set.
|
||||
#[test]
|
||||
fn a_series_with_no_result_after_a_closed_one_is_passed_over() {
|
||||
let map_grid = grid(0, 60_000, 1);
|
||||
@@ -1823,6 +1831,68 @@ fn a_series_with_no_result_after_a_closed_one_is_passed_over() {
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty closing chunk of one series does not close a label set another series is still filling.
|
||||
///
|
||||
/// `abs` drops the metric name, so `a{job="x"}` and `b{job="x"}` are one label set to the map. `b`
|
||||
/// holds no row at all: its closing chunk closes a series that contributed nothing, so it must not
|
||||
/// close the set `a` opened, or `a`'s continuation would be refused for a label set that `a` itself
|
||||
/// never closed. Two series back to back are what contiguous input hands over, so this is a stream
|
||||
/// the runtime executes rather than an ordering it refuses.
|
||||
#[test]
|
||||
fn an_empty_closing_chunk_of_another_series_does_not_close_a_label_set() {
|
||||
let map_grid = grid(0, 60_000, 2);
|
||||
let tree = NativeTree::try_new(map(
|
||||
evaluated_input(Arc::clone(&map_grid)),
|
||||
vec![MathOp::Abs],
|
||||
))
|
||||
.unwrap();
|
||||
let schema = evaluated_schema(Arc::clone(&map_grid));
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
|
||||
// `a` opens the label set and has not closed it when `b` closes an empty series onto it, and the
|
||||
// chunk that closes `a` itself comes last.
|
||||
let chunks = [
|
||||
evaluated_chunk_of(&schema, named_metric("a", "x"), 0, vec![Some(-1.0)], false),
|
||||
evaluated_chunk_of(&schema, named_metric("b", "x"), 0, vec![], true),
|
||||
evaluated_chunk_of(&schema, named_metric("a", "x"), 1, vec![Some(-2.0)], true),
|
||||
];
|
||||
let steps = chunks.len();
|
||||
let mut handed = Vec::new();
|
||||
for (step, chunk) in chunks.into_iter().enumerate() {
|
||||
execution.push_evaluated(chunk).unwrap();
|
||||
while let Some(chunk) = execution
|
||||
.next_chunk()
|
||||
.unwrap_or_else(|error| panic!("step {step} was refused: {error}"))
|
||||
{
|
||||
handed.push(chunk);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
handed.len(),
|
||||
steps,
|
||||
"the map did not consume all {steps} chunks it was given: {:?}",
|
||||
decided(&handed)
|
||||
);
|
||||
let last = handed.last().expect("the map handed nothing on");
|
||||
assert!(last.is_last, "the final output does not close its series");
|
||||
// `abs` drops the metric name, so the label set the output closes is `{job="x"}`.
|
||||
let mut mapped = named_metric("a", "x");
|
||||
mapped.remove("__name__");
|
||||
assert_eq!(
|
||||
last.series.metric, mapped,
|
||||
"the final output closes a label set other than the one `abs` leaves of `a{{job=\"x\"}}`"
|
||||
);
|
||||
assert_eq!(
|
||||
(series_start(last), sample_values(last)),
|
||||
(1, vec![Some(2.0)]),
|
||||
"the value `a` held at position 1 did not survive"
|
||||
);
|
||||
assert!(
|
||||
execution.finish().unwrap().is_empty(),
|
||||
"the map still held a chunk when the input ended"
|
||||
);
|
||||
}
|
||||
|
||||
/// Two series that each produce no value at all are not a collision: both answer empty, and the
|
||||
/// node hands nothing on rather than refusing the second one.
|
||||
///
|
||||
@@ -1958,10 +2028,10 @@ fn a_sparse_chunk_is_read_at_the_positions_it_holds_through_window_map_window()
|
||||
fn a_sparse_chunk_occupies_only_the_positions_it_holds() {
|
||||
let map_grid = grid(0, 60_000, 8);
|
||||
let tree = NativeTree::try_new(map(
|
||||
evaluated_input_with_ordering(Arc::clone(&map_grid), SeriesOrdering::Interleaved),
|
||||
evaluated_input(Arc::clone(&map_grid)),
|
||||
vec![MathOp::Abs],
|
||||
))
|
||||
.expect("a map-only tree is legal over an interleaved input");
|
||||
.unwrap();
|
||||
let schema = evaluated_schema(Arc::clone(&map_grid));
|
||||
|
||||
let mut execution = executor(&tree, pool(), &ExecutionPlanMetricsSet::new());
|
||||
@@ -2527,7 +2597,7 @@ fn a_series_whose_axis_and_fields_disagree_is_rejected() {
|
||||
|
||||
let mut emitted = Emitted::default();
|
||||
let error = emitted
|
||||
.occupy(&NodePath::root(), &chunk)
|
||||
.occupy(&NodePath::root(), Some(&chunk.series.metric), &chunk)
|
||||
.expect_err("a field with fewer samples than the axis places rows was accepted");
|
||||
let message = error.to_string();
|
||||
assert!(
|
||||
@@ -2543,8 +2613,8 @@ fn a_series_whose_axis_and_fields_disagree_is_rejected() {
|
||||
/// The `History` of a label set lives in the map's slot, so a `size_of::<History>()` term in the
|
||||
/// entry's own figure billed one header twice: once here and once in the capacity term beside it.
|
||||
/// The two assertions below are the definition of the whole figure -- the labels, the vectors and
|
||||
/// their runs, and one label-set slot plus one history slot per entry -- so either header counted
|
||||
/// in the wrong place fails them.
|
||||
/// their runs, the owner's name and tags, and one label-set slot plus one history slot per entry --
|
||||
/// so either header counted in the wrong place fails them.
|
||||
#[test]
|
||||
fn a_history_entry_is_charged_once_for_its_header() {
|
||||
let grid = grid(0, 10_000, 2);
|
||||
@@ -2553,15 +2623,32 @@ fn a_history_entry_is_charged_once_for_its_header() {
|
||||
let labels = label_set(&metric);
|
||||
// Not the closing chunk: a closed label set drops its runs, and the runs are what the entry's
|
||||
// own figure is.
|
||||
let chunk = evaluated_chunk_of(&schema, metric, 0, vec![Some(1.0), Some(2.0)], false);
|
||||
let chunk = evaluated_chunk_of(
|
||||
&schema,
|
||||
metric.clone(),
|
||||
0,
|
||||
vec![Some(1.0), Some(2.0)],
|
||||
false,
|
||||
);
|
||||
|
||||
let mut emitted = Emitted::default();
|
||||
emitted.occupy(&NodePath::root(), &chunk).unwrap();
|
||||
emitted
|
||||
.occupy(&NodePath::root(), Some(&metric), &chunk)
|
||||
.unwrap();
|
||||
|
||||
let history = emitted
|
||||
.sets
|
||||
.get(&labels)
|
||||
.expect("the chunk recorded its label set");
|
||||
// The owner the entry keeps is the series that filled it, so its name and each tag cost the
|
||||
// same bytes the label set of the chunk did.
|
||||
let owner = metric.name().map_or(0, |name| name.len())
|
||||
+ std::mem::size_of_val(metric.tags())
|
||||
+ metric
|
||||
.tags()
|
||||
.iter()
|
||||
.map(|tag| tag.name.len() + tag.value.len())
|
||||
.sum::<usize>();
|
||||
let entry = label_bytes(&labels)
|
||||
.saturating_add(history.fields.capacity() * std::mem::size_of::<Vec<Run>>())
|
||||
.saturating_add(
|
||||
@@ -2570,7 +2657,13 @@ fn a_history_entry_is_charged_once_for_its_header() {
|
||||
.iter()
|
||||
.map(|runs| runs.capacity() * std::mem::size_of::<Run>())
|
||||
.sum::<usize>(),
|
||||
);
|
||||
)
|
||||
.saturating_add(owner);
|
||||
assert_eq!(
|
||||
history.owner.as_ref(),
|
||||
Some(&metric),
|
||||
"the entry does not name the series that filled it"
|
||||
);
|
||||
assert!(
|
||||
history.fields.iter().any(|runs| !runs.is_empty()),
|
||||
"the entry kept no run, so it has no heap of its own to charge for"
|
||||
@@ -2578,8 +2671,8 @@ fn a_history_entry_is_charged_once_for_its_header() {
|
||||
assert_eq!(
|
||||
emitted.entries,
|
||||
entry,
|
||||
"the entry's own bytes are not its labels and runs alone: a `History` header in the figure \
|
||||
would add the {} bytes its slot already charges",
|
||||
"the entry's own bytes are not its labels, its runs and its owner: a `History` header in \
|
||||
the figure would add the {} bytes its slot already charges",
|
||||
std::mem::size_of::<History>(),
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -2625,7 +2718,7 @@ fn a_sparse_series_that_moves_forward_is_appended_to_its_history() {
|
||||
false,
|
||||
);
|
||||
emitted
|
||||
.occupy(&path, &chunk)
|
||||
.occupy(&path, Some(&chunk.series.metric), &chunk)
|
||||
.unwrap_or_else(|error| panic!("position {position} was refused: {error}"));
|
||||
};
|
||||
// The runs stored under the one label set, which is what `absorb` keeps in position order.
|
||||
|
||||
Reference in New Issue
Block a user