mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 00:34:11 +00:00
feat(promql): name native result columns from expressions (#10231)
Use the statement expression for native value column names, resolve legacy value columns by type rather than by name, and reject label-function destination collisions instead of dropping a value. Keep matrix roots named after the whole statement, preserve an Int64 __tsid value, and propagate schema-collision errors from canonical boundaries. Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
@@ -377,8 +377,8 @@ struct FusedSeriesPayload {
|
||||
|
||||
impl FusedSeries {
|
||||
/// Serializes the native tree and the binding required to reconstruct this plan node.
|
||||
/// The PromQL text is included for diagnostics only and never drives execution; children
|
||||
/// remain ordinary Substrait inputs.
|
||||
/// The PromQL text identifies expression-named output fields; children remain ordinary
|
||||
/// Substrait inputs.
|
||||
///
|
||||
/// This is only the node's own encoding. Whether a planner is allowed to place this node on
|
||||
/// another machine is a separate decision, made by the planner and the transport, not here.
|
||||
@@ -497,6 +497,7 @@ impl FusedSeries {
|
||||
&node.tree,
|
||||
&node.output_tags()?,
|
||||
&node.binding.time_index,
|
||||
&node.promql,
|
||||
)?;
|
||||
}
|
||||
Ok(node)
|
||||
@@ -1051,6 +1052,22 @@ impl FusedSeries {
|
||||
/// from the whole tree - a match narrows them, `or` joins both sides' and a wrapper around
|
||||
/// either keeps them - and the output schema is where it records that.
|
||||
fn output_tags(&self) -> DataFusionResult<Vec<String>> {
|
||||
if matches!(self.tree.root().output, NativeType::Scalar(_)) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if self.format == FusedOutputFormat::PlainRows {
|
||||
// Roles are types here too: a value is a float or a native histogram, so every other
|
||||
// non-time column of a row table is one of this output's labels, whatever its name.
|
||||
return Ok(self
|
||||
.output_schema
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|field| {
|
||||
field.name() != &self.binding.time_index && field.data_type() == &DataType::Utf8
|
||||
})
|
||||
.map(|field| field.name().clone())
|
||||
.collect());
|
||||
}
|
||||
let field_names = self
|
||||
.tree
|
||||
.root()
|
||||
@@ -1732,6 +1749,7 @@ fn validate_plain_output_schema(
|
||||
tree: &NativeTree,
|
||||
tags: &[String],
|
||||
time: &str,
|
||||
promql: &str,
|
||||
) -> DataFusionResult<()> {
|
||||
let fields = match &tree.root().output {
|
||||
NativeType::Samples(s) => s.fields.to_vec(),
|
||||
@@ -1755,7 +1773,7 @@ fn validate_plain_output_schema(
|
||||
));
|
||||
}
|
||||
};
|
||||
classify_output(schema, &fields, tags, time)?;
|
||||
classify_output(schema, &fields, tags, time, promql)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1844,13 +1862,73 @@ fn scalar_root_fields(
|
||||
}])
|
||||
}
|
||||
|
||||
/// Whether a column of this type carries a field's samples.
|
||||
///
|
||||
/// A role is a type: a field holds floats or the canonical native histogram struct, a tag is
|
||||
/// `Utf8` and time is a timestamp. A name says nothing about which of those a column is.
|
||||
fn is_value_column(data_type: &DataType) -> bool {
|
||||
data_type == &DataType::Float64 || data_type == &native_histogram_value_type().as_arrow_type()
|
||||
}
|
||||
|
||||
/// Whether `names` name exactly the columns in `value_names`, one name per column.
|
||||
fn names_value_columns(value_names: &[&str], names: &[String]) -> bool {
|
||||
if names.len() != value_names.len() {
|
||||
return false;
|
||||
}
|
||||
let mut declared = names.iter().map(String::as_str).collect::<Vec<_>>();
|
||||
declared.sort_unstable();
|
||||
let mut carried = value_names.to_vec();
|
||||
carried.sort_unstable();
|
||||
declared == carried
|
||||
}
|
||||
|
||||
/// Names the fields in a row schema, in `series_fields` order.
|
||||
///
|
||||
/// A caller names a value column either after the expression that produced it or after the tree's
|
||||
/// field, so the typed value columns are what decide which: the expression names are used only
|
||||
/// when they name exactly those columns, and the field names when those do. Deciding by type keeps
|
||||
/// a tag that shares a field's name from being read as that field. The result stays in
|
||||
/// `series_fields` order because a name is bound back to its field by that index, so a reordered
|
||||
/// output still reads every column from its own field.
|
||||
fn output_field_names(
|
||||
output_schema: &SchemaRef,
|
||||
series_fields: &[FieldSpec],
|
||||
promql: &str,
|
||||
) -> Vec<String> {
|
||||
let value_names = output_schema
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|field| is_value_column(field.data_type()))
|
||||
.map(|field| field.name().as_str())
|
||||
.collect::<Vec<_>>();
|
||||
let expression_names = series_fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
if series_fields.len() == 1 {
|
||||
promql.to_string()
|
||||
} else {
|
||||
format!("{promql}.{}", field.name)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if names_value_columns(&value_names, &expression_names) {
|
||||
return expression_names;
|
||||
}
|
||||
series_fields
|
||||
.iter()
|
||||
.map(|field| field.name.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Classifies every output column, rejecting anything the conversion cannot produce.
|
||||
pub(crate) fn classify_output(
|
||||
output_schema: &SchemaRef,
|
||||
series_fields: &[FieldSpec],
|
||||
tags: &[String],
|
||||
time_column: &str,
|
||||
promql: &str,
|
||||
) -> DataFusionResult<Vec<OutputColumn>> {
|
||||
let field_names = output_field_names(output_schema, series_fields, promql);
|
||||
if output_schema
|
||||
.fields()
|
||||
.iter()
|
||||
@@ -1873,11 +1951,6 @@ pub(crate) fn classify_output(
|
||||
"tag column {tag:?} of FusedSeriesExec cannot be the time column"
|
||||
)));
|
||||
}
|
||||
if series_fields.iter().any(|field| field.name.as_ref() == tag) {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"tag column {tag:?} of FusedSeriesExec clashes with a series field"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut time_seen = false;
|
||||
@@ -1909,10 +1982,7 @@ pub(crate) fn classify_output(
|
||||
classification.push(OutputColumn::Time);
|
||||
continue;
|
||||
}
|
||||
if let Some(index) = series_fields
|
||||
.iter()
|
||||
.position(|field| field.name.as_ref() == name)
|
||||
{
|
||||
if let Some(index) = field_names.iter().position(|field| field == name) {
|
||||
if std::mem::replace(&mut field_seen[index], true) {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"duplicate field column {name:?} in FusedSeriesExec output schema"
|
||||
@@ -1977,7 +2047,7 @@ pub(crate) fn classify_output(
|
||||
if !seen {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"missing field column {:?} in FusedSeriesExec output schema",
|
||||
series_fields[index].name
|
||||
field_names[index]
|
||||
)));
|
||||
}
|
||||
}
|
||||
@@ -2485,7 +2555,7 @@ impl FusedSeriesExec {
|
||||
Vec::new()
|
||||
} else if scalar_root {
|
||||
let scalar_fields = scalar_root_fields(&output_schema, &tags, &time_column)?;
|
||||
classify_output(&output_schema, &scalar_fields, &tags, &time_column)?
|
||||
classify_output(&output_schema, &scalar_fields, &tags, &time_column, &promql)?
|
||||
} else {
|
||||
classify_output(
|
||||
&output_schema,
|
||||
@@ -2496,6 +2566,7 @@ impl FusedSeriesExec {
|
||||
.fields,
|
||||
&tags,
|
||||
&time_column,
|
||||
&promql,
|
||||
)?
|
||||
};
|
||||
let partition_count = inputs.first().map_or(1, |(input, _, _)| {
|
||||
|
||||
@@ -2599,6 +2599,149 @@ fn assert_decode_error(bytes: &[u8], expected: &str) {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renamed_reordered_value_columns_keep_field_identity_after_decode() {
|
||||
let tree = rate_tree_of(two_field_schema());
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new(format!("{PROMQL}.{OTHER}"), DataType::Float64, true),
|
||||
Field::new(HOST, DataType::Utf8, true),
|
||||
Field::new(
|
||||
TIME,
|
||||
DataType::Timestamp(TimeUnit::Millisecond, None),
|
||||
false,
|
||||
),
|
||||
Field::new(format!("{PROMQL}.{VALUE}"), DataType::Float64, true),
|
||||
]));
|
||||
let node = FusedSeries::try_new(
|
||||
Arc::from(PROMQL),
|
||||
tree.clone(),
|
||||
binding_of(&tree, two_field_schema(), vec![HOST.to_string()]),
|
||||
vec![input_plan(frame_schema_of(&two_field_schema()))],
|
||||
Arc::new(datafusion::common::DFSchema::try_from(schema.as_ref().clone()).unwrap()),
|
||||
)
|
||||
.unwrap();
|
||||
let decoded = FusedSeries::deserialize(&node.serialize().unwrap()).unwrap();
|
||||
assert_eq!(node.schema(), decoded.schema());
|
||||
let tags = decoded.output_tags().unwrap();
|
||||
assert_eq!(vec![HOST], tags);
|
||||
let classification =
|
||||
super::classify_output(&schema, &two_field_schema().fields, &tags, TIME, PROMQL).unwrap();
|
||||
let chunk = EvaluatedChunk::try_new(
|
||||
Series {
|
||||
metric: SeriesLabels::try_new(None, Vec::new()).unwrap(),
|
||||
time: TimeAxis::from_grid_spans(grid(), vec![GridSpan::new(0, 1)]).unwrap(),
|
||||
fields: SeriesFields::try_from_vec(vec![
|
||||
Samples::from_float(Float64Array::from(vec![10.0])),
|
||||
Samples::from_float(Float64Array::from(vec![20.0])),
|
||||
])
|
||||
.unwrap(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
let batch = series_to_record_batch(&classification, &tags, &schema, &chunk).unwrap();
|
||||
for (name, expected) in [(OTHER, 20.0), (VALUE, 10.0)] {
|
||||
let values = batch
|
||||
.column_by_name(&format!("{PROMQL}.{name}"))
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.unwrap();
|
||||
assert_eq!(expected, values.value(0));
|
||||
}
|
||||
}
|
||||
|
||||
/// A tag that carries the expression's name is a tag, not the statement's value.
|
||||
///
|
||||
/// This is an old-style row schema: the statement is `host`, the schema names its one value
|
||||
/// `field_0` and also carries the `host` tag. Deciding the value by whether a name occurs
|
||||
/// anywhere in the schema would pick the tag as the value; the typed columns put the value at
|
||||
/// `field_0` and the tag at `host`.
|
||||
#[test]
|
||||
fn a_tag_named_after_the_expression_is_not_the_value() {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new(
|
||||
TIME,
|
||||
DataType::Timestamp(TimeUnit::Millisecond, None),
|
||||
false,
|
||||
),
|
||||
Field::new("field_0", DataType::Float64, true),
|
||||
Field::new(HOST, DataType::Utf8, true),
|
||||
]));
|
||||
let fields = [FieldSpec {
|
||||
name: "field_0".into(),
|
||||
}];
|
||||
let classification =
|
||||
super::classify_output(&schema, &fields, &[HOST.to_string()], TIME, HOST).unwrap();
|
||||
assert!(
|
||||
matches!(
|
||||
classification[1],
|
||||
OutputColumn::Field {
|
||||
index: 0,
|
||||
kind: FieldKind::Float
|
||||
}
|
||||
),
|
||||
"{classification:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(classification[2], OutputColumn::Tag(0)),
|
||||
"{classification:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A reordered output still binds every value column to its own field.
|
||||
///
|
||||
/// The expression names are matched as a set against the typed value columns, and the result is
|
||||
/// kept in `series_fields` order - here `other` before `value` - because that order is what the
|
||||
/// classification indexes the chunk's fields by. Names taken in output order would bind the first
|
||||
/// column to `value`.
|
||||
#[test]
|
||||
fn reordered_expression_names_bind_fields_in_series_order() {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new(
|
||||
TIME,
|
||||
DataType::Timestamp(TimeUnit::Millisecond, None),
|
||||
false,
|
||||
),
|
||||
Field::new(format!("{PROMQL}.{OTHER}"), DataType::Float64, true),
|
||||
Field::new(HOST, DataType::Utf8, true),
|
||||
Field::new(format!("{PROMQL}.{VALUE}"), DataType::Float64, true),
|
||||
]));
|
||||
let series_schema = two_field_schema();
|
||||
let classification = super::classify_output(
|
||||
&schema,
|
||||
&series_schema.fields,
|
||||
&[HOST.to_string()],
|
||||
TIME,
|
||||
PROMQL,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(
|
||||
classification[1],
|
||||
OutputColumn::Field {
|
||||
index: 1,
|
||||
kind: FieldKind::Float
|
||||
}
|
||||
),
|
||||
"{classification:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(classification[2], OutputColumn::Tag(0)),
|
||||
"{classification:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
classification[3],
|
||||
OutputColumn::Field {
|
||||
index: 0,
|
||||
kind: FieldKind::Float
|
||||
}
|
||||
),
|
||||
"{classification:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_rows_decode_rejects_invalid_schemas() {
|
||||
let node = fused(rate_tree());
|
||||
@@ -2649,16 +2792,62 @@ fn plain_rows_decode_rejects_invalid_schemas() {
|
||||
false,
|
||||
),
|
||||
Field::new(VALUE, DataType::Float64, true),
|
||||
// A column's role is its type: a float here is not a tag, so this payload names a
|
||||
// column the conversion cannot produce.
|
||||
Field::new(HOST, DataType::Float64, true),
|
||||
Field::new(NAME, DataType::Utf8, true),
|
||||
]),
|
||||
"must be Utf8",
|
||||
"unknown column",
|
||||
),
|
||||
];
|
||||
for (schema, expected) in malformed_schemas {
|
||||
assert_decode_error(&mutate_schema(&bytes, schema), expected);
|
||||
}
|
||||
|
||||
let renamed = mutate_envelope_payload(&bytes, |payload| {
|
||||
let mut fields = payload.output_schema.fields().to_vec();
|
||||
let index = fields
|
||||
.iter()
|
||||
.position(|field| field.name() == VALUE)
|
||||
.unwrap();
|
||||
fields[index] = Arc::new(Field::new(PROMQL, DataType::Float64, true));
|
||||
payload.output_schema = Schema::new(fields);
|
||||
});
|
||||
assert!(FusedSeries::deserialize(&renamed).is_ok());
|
||||
// A value column is identified by type, so this payload types its one float column as neither
|
||||
// a float nor a native histogram and names no value column the tree can read; a float value
|
||||
// column that cannot be null is refused for its nullability, because a grid point may be
|
||||
// missing.
|
||||
for (data_type, nullable, expected) in [
|
||||
(DataType::Int64, true, "unknown column"),
|
||||
(DataType::Float64, false, "must be nullable"),
|
||||
] {
|
||||
let malformed = mutate_envelope_payload(&renamed, |payload| {
|
||||
let mut fields = payload.output_schema.fields().to_vec();
|
||||
let index = fields
|
||||
.iter()
|
||||
.position(|field| field.name() == PROMQL)
|
||||
.unwrap();
|
||||
fields[index] = Arc::new(Field::new(PROMQL, data_type, nullable));
|
||||
payload.output_schema = Schema::new(fields);
|
||||
});
|
||||
assert_decode_error(&malformed, expected);
|
||||
}
|
||||
|
||||
// A scalar's value column is its one non-time column, so the wrong type is reported as a kind
|
||||
// the conversion cannot produce.
|
||||
let scalar_wrong_kind = mutate_envelope_payload(&scalar_bytes, |payload| {
|
||||
let mut fields = payload.output_schema.fields().to_vec();
|
||||
let index = fields
|
||||
.iter()
|
||||
.position(|field| field.name() != TIME)
|
||||
.expect("a scalar carries one value column");
|
||||
let name = fields[index].name().clone();
|
||||
fields[index] = Arc::new(Field::new(name, DataType::Int64, true));
|
||||
payload.output_schema = Schema::new(fields);
|
||||
});
|
||||
assert_decode_error(&scalar_wrong_kind, "must be Float64");
|
||||
|
||||
let scalar_decoded = FusedSeries::deserialize(&scalar_bytes).unwrap();
|
||||
assert_eq!(scalar_decoded.schema(), scalar.schema());
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
//! nodes, recursing into a cut that wraps a whole-vector consumer.
|
||||
//!
|
||||
//! Everything here reads the bound tree alone. The columns a node publishes are the planner's
|
||||
//! answer, so the builders ask for them through [`OutputTags`] and [`RootSchema`] instead of
|
||||
//! deriving them a second time.
|
||||
//! answer, so the builders ask for them through [`OutputTags`], [`FieldNames`] and [`RootSchema`]
|
||||
//! instead of deriving them a second time.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -56,11 +56,20 @@ use crate::vector_match::Cardinality;
|
||||
/// rather than deriving the columns a second time.
|
||||
pub type OutputTags = fn(&NativeNode, &[Vec<String>]) -> Option<Vec<String>>;
|
||||
|
||||
/// The row schema a statement root publishes, or `None` when its columns cannot be named once.
|
||||
/// Derives the value column names a statement's expression gives the compiled `fields`.
|
||||
///
|
||||
/// Like [`OutputTags`], this is the planner's answer: a segment publishes the values its tree
|
||||
/// computes under the names the statement gave them, so the builders ask through this leaf instead
|
||||
/// of publishing the compiler's own field names. The tree alone does not carry the statement's
|
||||
/// expression, which is why the expression text is an input rather than something derived here.
|
||||
pub type FieldNames = fn(&str, &[String]) -> Vec<String>;
|
||||
|
||||
/// The row schema a statement root publishes for `promql`, or `None` when its columns cannot be
|
||||
/// named once.
|
||||
///
|
||||
/// Like [`OutputTags`], this is the caller's answer: a segment that publishes rows publishes
|
||||
/// exactly the columns the planner would have published for the same tree.
|
||||
pub type RootSchema = fn(&NativeTree, &[Vec<String>], &str, FieldKind) -> Option<DFSchemaRef>;
|
||||
/// exactly the columns the planner would have published for the same statement.
|
||||
pub type RootSchema = fn(&str, &NativeTree, &[Vec<String>], &str, FieldKind) -> Option<DFSchemaRef>;
|
||||
|
||||
/// Whether `node` reads samples point-wise, so that one partitioned execution is one answer.
|
||||
///
|
||||
@@ -496,8 +505,8 @@ pub const NATIVE_ORDERING: SeriesOrdering = SeriesOrdering::Contiguous;
|
||||
|
||||
/// The columns a [`FusedSeries`] node produces.
|
||||
///
|
||||
/// With `experimental_native_promql` enabled, a value column is named after its compiled root
|
||||
/// field (for example, `field_0`), not legacy planner expression text. Legacy headers in
|
||||
/// With `experimental_native_promql` enabled, a value column is named by the caller's [`FieldNames`]
|
||||
/// answer for the statement's expression, not legacy planner expression text. Legacy headers in
|
||||
/// `tests/cases/**` are unaffected because the native route is opt-in and off by default.
|
||||
///
|
||||
/// The shape is the one the existing unary pipeline consumes — the time index, then the series
|
||||
@@ -623,6 +632,7 @@ pub fn canonical_boundary_stages(
|
||||
format: FusedOutputFormat,
|
||||
root_kind: FieldKind,
|
||||
output_tags_of: OutputTags,
|
||||
field_names: FieldNames,
|
||||
) -> Option<(LogicalPlan, Vec<String>)> {
|
||||
let NativeExpr::Merge { input } = &tree.root().expr else {
|
||||
return None;
|
||||
@@ -644,7 +654,12 @@ pub fn canonical_boundary_stages(
|
||||
// declares it exactly as the other rows layouts do. The tags stay the walk's own: they name the
|
||||
// labels the frames carry, and the binding below is built from them.
|
||||
let row_tags = with_row_name_column(tree.root(), output_tags.clone());
|
||||
let output_schema = fused_output_schema(&fields, &row_tags, time_index, root_kind).ok()?;
|
||||
// The values are named after the expression, as in every other rows layout of this statement.
|
||||
// A schema this boundary cannot declare declines here: the caller's error type is not one this
|
||||
// crate can build, so a name collision is reported by declining rather than as an internal
|
||||
// error, and the caller keeps the plan it would have chosen.
|
||||
let value_names = field_names(&promql, &fields);
|
||||
let output_schema = fused_output_schema(&value_names, &row_tags, time_index, root_kind).ok()?;
|
||||
|
||||
// The producer: the statement's own tree, run per partition, exporting a frame. It runs above
|
||||
// the regions, where its router collects each label set from every region's contribution.
|
||||
@@ -766,6 +781,7 @@ fn build_evaluated_segment(
|
||||
time_index: &str,
|
||||
root_kind: FieldKind,
|
||||
output_tags_of: OutputTags,
|
||||
field_names: FieldNames,
|
||||
) -> Option<(LogicalPlan, Vec<String>)> {
|
||||
let mut sources = Vec::new();
|
||||
collect_segment_inputs(node, &mut sources);
|
||||
@@ -829,6 +845,7 @@ fn build_evaluated_segment(
|
||||
FusedOutputFormat::SeriesFrame,
|
||||
root_kind,
|
||||
output_tags_of,
|
||||
field_names,
|
||||
)?;
|
||||
return Some((plan, tags));
|
||||
}
|
||||
@@ -959,7 +976,8 @@ fn build_inner_segment(
|
||||
/// per-series prefixes do not need that gather, so each becomes a segment of its own and keeps the
|
||||
/// partitions it was scanned with. Global consumers and canonical merges are cut in their original
|
||||
/// order, and recursive segments publish frames for their parents to consume. Only the statement
|
||||
/// root publishes rows.
|
||||
/// root publishes rows, and it does so under the statement's own value names: [`FieldNames`]
|
||||
/// answers those, and the rows schema asks [`RootSchema`] for the rest of the layout.
|
||||
#[allow(clippy::too_many_arguments, clippy::type_complexity)]
|
||||
pub fn segment_tree(
|
||||
promql: Arc<str>,
|
||||
@@ -977,6 +995,7 @@ pub fn segment_tree(
|
||||
root_kind: FieldKind,
|
||||
remote_raw_boundary: bool,
|
||||
output_tags_of: OutputTags,
|
||||
field_names: FieldNames,
|
||||
root_schema: RootSchema,
|
||||
) -> Option<LogicalPlan> {
|
||||
// Only a tree the analyzer gathers is worth splitting: its per-series prefixes then run where
|
||||
@@ -1061,6 +1080,7 @@ pub fn segment_tree(
|
||||
root_kind,
|
||||
remote_raw_boundary,
|
||||
output_tags_of,
|
||||
field_names,
|
||||
root_schema,
|
||||
);
|
||||
match plan {
|
||||
@@ -1096,6 +1116,7 @@ pub fn segment_tree(
|
||||
time_index,
|
||||
root_kind,
|
||||
output_tags_of,
|
||||
field_names,
|
||||
)
|
||||
});
|
||||
let (plan, cut_tags) = built?;
|
||||
@@ -1164,6 +1185,7 @@ pub fn segment_tree(
|
||||
format,
|
||||
root_kind,
|
||||
output_tags_of,
|
||||
field_names,
|
||||
)
|
||||
.map(|(plan, _)| plan);
|
||||
}
|
||||
@@ -1176,7 +1198,9 @@ pub fn segment_tree(
|
||||
return None;
|
||||
}
|
||||
let output_schema = match format {
|
||||
FusedOutputFormat::PlainRows => root_schema(&parent, &child_tags, time_index, root_kind)?,
|
||||
FusedOutputFormat::PlainRows => {
|
||||
root_schema(&promql, &parent, &child_tags, time_index, root_kind)?
|
||||
}
|
||||
FusedOutputFormat::SeriesFrame => {
|
||||
let Ok(frame) = FusedSeries::series_frame_schema(&parent) else {
|
||||
return None;
|
||||
|
||||
+19
-21
@@ -1866,9 +1866,8 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A statement that reads no table plans natively, and there is no region to place anything in:
|
||||
/// the frontend evaluates the fused node under the native column names rather than the ones the
|
||||
/// existing planner derives from an expression.
|
||||
/// A statement that reads no table plans natively over the wire, where there is no region to
|
||||
/// place anything in: the frontend evaluates the fused node with expression-named values.
|
||||
#[tokio::test]
|
||||
async fn test_remote_native_promql_plans_a_data_free_statement_natively() {
|
||||
let engine = create_promql_test_engine_with(
|
||||
@@ -1879,35 +1878,26 @@ mod tests {
|
||||
false,
|
||||
);
|
||||
let query_ctx = native_discovery_query_ctx();
|
||||
for (query, expected_columns, expected_values, expected_labels) in [
|
||||
// `time()` answers the evaluation grid in seconds.
|
||||
(
|
||||
"time()",
|
||||
&["time", "value"][..],
|
||||
&[0.0, 5.0, 10.0][..],
|
||||
&[][..],
|
||||
),
|
||||
for (query, expected_values, expected_labels, name_column) in [
|
||||
// `time()` answers the evaluation grid in seconds. It is a scalar root: it publishes
|
||||
// one value and no series, so its rows layout declares no name column.
|
||||
("time()", &[0.0, 5.0, 10.0][..], &[][..], false),
|
||||
// `vector(1)` is a samples root: its rows layout declares the per-row name column,
|
||||
// which names no row because the vector carries no series identity.
|
||||
(
|
||||
"vector(1)",
|
||||
&["time", "value", "__name__"][..],
|
||||
&[1.0, 1.0, 1.0][..],
|
||||
&[][..],
|
||||
),
|
||||
("vector(1)", &[1.0, 1.0, 1.0][..], &[][..], true),
|
||||
(
|
||||
r#"label_replace(vector(1), "host", "host1", "", "")"#,
|
||||
&["time", "value", "host", "__name__"][..],
|
||||
&[1.0, 1.0, 1.0][..],
|
||||
&[("host", "host1")][..],
|
||||
true,
|
||||
),
|
||||
// A synthesized `__tsid` is published under the statement's own columns: the stripping
|
||||
// projection above the node is read through rather than mistaken for a remote plan.
|
||||
(
|
||||
r#"label_replace(vector(1), "__tsid", "x", "", "")"#,
|
||||
&["time", "value", "__name__"][..],
|
||||
&[1.0, 1.0, 1.0][..],
|
||||
&[][..],
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
|
||||
@@ -1917,7 +1907,15 @@ mod tests {
|
||||
!display.contains("greptime_value"),
|
||||
"{query} keeps the native value column: {display}"
|
||||
);
|
||||
// A data-free statement has no input at all, so nothing is scanned or left empty.
|
||||
// A data-free statement has no input at all, so nothing is scanned or left empty. The
|
||||
// value is named after the expression, and a samples root's rows keep the name column
|
||||
// last.
|
||||
let expression = promql_parser::parser::parse(query).unwrap().to_string();
|
||||
let mut expected_columns = vec!["time", expression.as_str()];
|
||||
expected_columns.extend(expected_labels.iter().map(|(name, _)| *name));
|
||||
if name_column {
|
||||
expected_columns.push("__name__");
|
||||
}
|
||||
assert_eq!(
|
||||
expected_columns,
|
||||
plan.schema()
|
||||
@@ -1931,7 +1929,7 @@ mod tests {
|
||||
assert_regionless_physical_plan(&physical, query);
|
||||
assert_eq!(
|
||||
expected_values,
|
||||
numeric_values(&batches, "value").as_slice(),
|
||||
numeric_values(&batches, &expression).as_slice(),
|
||||
"{query}: {batches:?}"
|
||||
);
|
||||
for (label, value) in expected_labels {
|
||||
|
||||
@@ -58,6 +58,15 @@ pub enum Error {
|
||||
location: Location,
|
||||
},
|
||||
|
||||
#[snafu(display(
|
||||
"duplicate native PromQL output column name(s): {names}; time index, value and tag columns must have distinct names"
|
||||
))]
|
||||
DuplicateOutputColumns {
|
||||
names: String,
|
||||
#[snafu(implicit)]
|
||||
location: Location,
|
||||
},
|
||||
|
||||
#[snafu(display("Unexpected plan or expression: {}", desc))]
|
||||
UnexpectedPlanExpr {
|
||||
desc: String,
|
||||
@@ -281,7 +290,8 @@ impl ErrorExt for Error {
|
||||
| SystemTimeOutOfRange { .. }
|
||||
| AtModifierTimestampOutOfRange { .. }
|
||||
| InvalidRegularExpression { .. }
|
||||
| InvalidDestinationLabelName { .. } => StatusCode::InvalidArguments,
|
||||
| InvalidDestinationLabelName { .. }
|
||||
| DuplicateOutputColumns { .. } => StatusCode::InvalidArguments,
|
||||
|
||||
UnknownTable { .. } => StatusCode::Internal,
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ use datafusion_expr::expr_fn::when;
|
||||
use datafusion_expr::utils::{conjunction, disjunction};
|
||||
use datafusion_expr::{ExprSchemable, Literal, SortExpr, TableSource, ident, lit};
|
||||
use datafusion_functions::core::coalesce;
|
||||
use datatypes::arrow::datatypes::{DataType as ArrowDataType, TimeUnit as ArrowTimeUnit};
|
||||
use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit as ArrowTimeUnit};
|
||||
use datatypes::data_type::{ConcreteDataType, DataType as GreptimeDataType};
|
||||
use itertools::Itertools;
|
||||
use once_cell::sync::Lazy;
|
||||
@@ -1435,13 +1435,11 @@ impl PromPlanner {
|
||||
let right_ctx = self.ctx.clone();
|
||||
(left, left_ctx, right, right_ctx)
|
||||
};
|
||||
let left_field_columns = left_context.field_columns.clone();
|
||||
let left_time_index_column = left_context.time_index_column.clone();
|
||||
self.ctx = left_context.clone();
|
||||
let mut left_table_ref = self
|
||||
.table_ref()
|
||||
.unwrap_or_else(|_| TableReference::bare(""));
|
||||
let right_field_columns = right_context.field_columns.clone();
|
||||
let right_time_index_column = right_context.time_index_column.clone();
|
||||
self.ctx = right_context.clone();
|
||||
let mut right_table_ref = self
|
||||
@@ -1450,7 +1448,8 @@ impl PromPlanner {
|
||||
|
||||
// Both operands are planned first because only the planned contexts tell a tag
|
||||
// from a value field. The rewrite merely adds matchers to a selector, so the
|
||||
// table reference, time index and field columns captured above stay valid.
|
||||
// table reference and time index captured above stay valid. The field columns are
|
||||
// read after this rewrite, because an operand it re-plans brings its own.
|
||||
if let Some(rewritten) = matching_filters::propagate(
|
||||
binary_expr,
|
||||
&left_context.tag_columns,
|
||||
@@ -1481,6 +1480,8 @@ impl PromPlanner {
|
||||
self.ctx = right_context.clone();
|
||||
}
|
||||
|
||||
let left_field_columns = left_context.field_columns.clone();
|
||||
let right_field_columns = right_context.field_columns.clone();
|
||||
let left_is_empty_metric = Self::is_empty_metric(&left_input);
|
||||
let right_is_empty_metric = Self::is_empty_metric(&right_input);
|
||||
|
||||
@@ -4019,12 +4020,28 @@ impl PromPlanner {
|
||||
query_engine_state,
|
||||
)?;
|
||||
|
||||
// Reserve the current field columns except the `dst_label`.
|
||||
for value in &self.ctx.field_columns {
|
||||
if *value != dst_label {
|
||||
let expr = DfExpr::Column(Column::from_name(value));
|
||||
exprs.push(expr);
|
||||
// A destination label is a tag, and a tag is published under its own name. A
|
||||
// value field of that name would have to be dropped or renamed to make room, and
|
||||
// neither is what the statement says, so the collision is the statement's error
|
||||
// rather than a value quietly going missing.
|
||||
if self
|
||||
.ctx
|
||||
.field_columns
|
||||
.iter()
|
||||
.any(|field| field == &dst_label)
|
||||
{
|
||||
return UnsupportedExprSnafu {
|
||||
name: format!(
|
||||
"label_join destination label {dst_label:?} collides with a value field"
|
||||
),
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
|
||||
// Reserve the current field columns.
|
||||
for value in &self.ctx.field_columns {
|
||||
let expr = DfExpr::Column(Column::from_name(value));
|
||||
exprs.push(expr);
|
||||
}
|
||||
|
||||
// Remove it from tag columns if exists to avoid duplicated column names
|
||||
@@ -4040,12 +4057,10 @@ impl PromPlanner {
|
||||
if let Some((replace_expr, dst_label)) =
|
||||
self.build_regexp_replace_label_expr(&mut other_input_exprs, input_schema)?
|
||||
{
|
||||
// Reserve the current field columns except the `dst_label`.
|
||||
// Reserve the current field columns.
|
||||
for value in &self.ctx.field_columns {
|
||||
if *value != dst_label {
|
||||
let expr = DfExpr::Column(Column::from_name(value));
|
||||
exprs.push(expr);
|
||||
}
|
||||
let expr = DfExpr::Column(Column::from_name(value));
|
||||
exprs.push(expr);
|
||||
}
|
||||
|
||||
// Remove it from tag columns if exists to avoid duplicated column names
|
||||
@@ -4375,6 +4390,22 @@ impl PromPlanner {
|
||||
|
||||
// Validate the destination label name
|
||||
Self::validate_label_name(&dst_label)?;
|
||||
// A destination label is a tag, and a tag is published under its own name. A value field
|
||||
// of that name would have to be dropped or renamed to make room, so report the collision
|
||||
// before any no-op path below can silently decline the rewrite.
|
||||
if self
|
||||
.ctx
|
||||
.field_columns
|
||||
.iter()
|
||||
.any(|field| field == &dst_label)
|
||||
{
|
||||
return UnsupportedExprSnafu {
|
||||
name: format!(
|
||||
"label_replace destination label {dst_label:?} collides with a value field"
|
||||
),
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
let replacement = match other_input_exprs.pop_front() {
|
||||
Some(DfExpr::Literal(ScalarValue::Utf8(Some(r)), _)) => r,
|
||||
other => UnexpectedPlanExprSnafu {
|
||||
@@ -6668,11 +6699,18 @@ impl PromPlanner {
|
||||
|
||||
fn strip_tsid_column(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
|
||||
let schema = plan.schema();
|
||||
if !schema
|
||||
.fields()
|
||||
.iter()
|
||||
.any(|field| field.name() == DATA_SCHEMA_TSID_COLUMN_NAME)
|
||||
{
|
||||
// A series identifier is the internal `UInt64` key or the `Utf8` label synthesized from
|
||||
// it, and only those two representations are stripped. The same name on any other column
|
||||
// is the statement's own answer - `count(m)` aliased to `__tsid` answers an `Int64`, and a
|
||||
// value is `Float64` or the canonical native histogram struct - so a name is not a role.
|
||||
fn is_internal_tsid(field: &Field) -> bool {
|
||||
field.name() == DATA_SCHEMA_TSID_COLUMN_NAME
|
||||
&& matches!(
|
||||
field.data_type(),
|
||||
ArrowDataType::UInt64 | ArrowDataType::Utf8
|
||||
)
|
||||
}
|
||||
if !schema.fields().iter().any(|field| is_internal_tsid(field)) {
|
||||
return Ok(plan);
|
||||
}
|
||||
|
||||
@@ -6680,7 +6718,7 @@ impl PromPlanner {
|
||||
// the columns by their original qualified names.
|
||||
let project_exprs = schema
|
||||
.iter()
|
||||
.filter(|(_, field)| field.name() != DATA_SCHEMA_TSID_COLUMN_NAME)
|
||||
.filter(|(_, field)| !is_internal_tsid(field))
|
||||
.map(|(qualifier, field)| {
|
||||
DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone()))
|
||||
})
|
||||
|
||||
@@ -77,11 +77,11 @@ use promql_parser::parser::{
|
||||
EvalStmt, Expr as PromExpr, MatrixSelector, ParenExpr, SubqueryExpr, VectorSelector,
|
||||
};
|
||||
use session::context::{MetricDiscoveryScope, QueryContextRef};
|
||||
use snafu::ResultExt;
|
||||
use snafu::{ResultExt, ensure};
|
||||
use store_api::metric_engine_consts::{DATA_SCHEMA_TABLE_ID_COLUMN_NAME, METRIC_ENGINE_NAME};
|
||||
use store_api::region_engine::RawLabelSource;
|
||||
|
||||
use crate::promql::error::{DataFusionPlanningSnafu, Result};
|
||||
use crate::promql::error::{DataFusionPlanningSnafu, DuplicateOutputColumnsSnafu, Result};
|
||||
use crate::promql::native_compile::{
|
||||
InfoInput, bind_evaluated_binary, compile_selectors_with_grid, compile_statement_with_grid,
|
||||
info_input, statement_grid, subquery_child_grid,
|
||||
@@ -528,6 +528,8 @@ impl PromPlanner {
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
// The expression names the value columns, and it is the label the node reports.
|
||||
let expression = expr.to_string();
|
||||
// What a consumer of this composition reads, and the layout the node publishes. The row
|
||||
// layout names one column per value; the frame layout is the evaluated frame this tree's
|
||||
// own root declares, which is what a frame consumer decodes.
|
||||
@@ -542,8 +544,13 @@ impl PromPlanner {
|
||||
};
|
||||
let output_schema = match format {
|
||||
FusedOutputFormat::PlainRows => {
|
||||
let Some(schema) =
|
||||
Self::fused_root_schema(&tree, &source_tags, &left_time, root_kind)
|
||||
let Some(schema) = Self::fused_root_schema(
|
||||
&expression,
|
||||
&tree,
|
||||
&source_tags,
|
||||
&left_time,
|
||||
root_kind,
|
||||
)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -562,7 +569,7 @@ impl PromPlanner {
|
||||
}
|
||||
};
|
||||
let Ok(node) = FusedSeries::try_new_with_format(
|
||||
Arc::from(expr.to_string()),
|
||||
Arc::from(expression.as_str()),
|
||||
tree,
|
||||
FusedSeriesBinding {
|
||||
grid: Arc::clone(&left_grid),
|
||||
@@ -586,8 +593,8 @@ impl PromPlanner {
|
||||
|
||||
/// Plans `prom_expr` as one native node, or returns `None` to keep the existing plan.
|
||||
///
|
||||
/// The compiler's bound tree and selector schemas determine the native output names; the existing
|
||||
/// planner is not consulted for capability or schema approval.
|
||||
/// The expression names the value columns; the compiler's bound tree and selector schemas
|
||||
/// determine their values. The existing planner is not consulted for capability or schema approval.
|
||||
///
|
||||
/// The route is offered a whole expression because the compiler folds a range function and the math
|
||||
/// functions over it into one tree: offering a sub-expression would compile a tree that computes
|
||||
@@ -606,6 +613,10 @@ impl PromPlanner {
|
||||
if !self.native_fused {
|
||||
return Ok(None);
|
||||
}
|
||||
// The expression names the value columns of every rows layout this attempt publishes, and the
|
||||
// node's own label follows the same text. It is the whole statement's expression even when a
|
||||
// subquery root compiles its inner one, because that is the statement the reader wrote.
|
||||
let expression = prom_expr.to_string();
|
||||
// A range selector returns raw samples, which have no evaluation grid to bind. A subquery root
|
||||
// instead returns its inner expression on the subquery's child grid, and a range query has no grid
|
||||
// to bind a matrix root to.
|
||||
@@ -689,7 +700,13 @@ impl PromPlanner {
|
||||
let Some(values) = native_root_fields(tree.root()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let time_index = if values
|
||||
// A label named `time` is the one collision the default time column can have: the
|
||||
// value columns are named after the expression, so they never take that name. The
|
||||
// time column is this route's own synthetic name - no label or value is renamed - so
|
||||
// a statement whose output already holds a `time` column publishes its grid under the
|
||||
// first free `native_time_N` instead of dropping or renaming the label.
|
||||
let value_names = expression_field_names(&expression, &values);
|
||||
let time_index = if value_names
|
||||
.iter()
|
||||
.chain(&output_tags)
|
||||
.any(|name| name == DEFAULT_TIME_INDEX_COLUMN)
|
||||
@@ -697,7 +714,7 @@ impl PromPlanner {
|
||||
(0usize..)
|
||||
.map(|index| format!("native_time_{index}"))
|
||||
.find(|candidate| {
|
||||
!values
|
||||
!value_names
|
||||
.iter()
|
||||
.chain(&output_tags)
|
||||
.any(|name| name == candidate)
|
||||
@@ -717,6 +734,7 @@ impl PromPlanner {
|
||||
// A data-free tree reads no input, so every field it answers is a float.
|
||||
let output_schema = match format {
|
||||
FusedOutputFormat::PlainRows => fused_output_schema(
|
||||
&expression,
|
||||
&values,
|
||||
&with_row_name_column(tree.root(), output_tags.clone()),
|
||||
&time_index,
|
||||
@@ -732,7 +750,7 @@ impl PromPlanner {
|
||||
}
|
||||
};
|
||||
let node = FusedSeries::try_new_with_format(
|
||||
Arc::from(prom_expr.to_string()),
|
||||
Arc::from(expression.as_str()),
|
||||
tree,
|
||||
FusedSeriesBinding {
|
||||
grid: Arc::clone(&grid),
|
||||
@@ -1466,7 +1484,7 @@ impl PromPlanner {
|
||||
if selectors.len() == 1
|
||||
&& matches!(&tree.root().expr, NativeExpr::Merge { input } if pointwise_operand(input) && canonical_local_prefix(input) && !contains_global_consumer(input))
|
||||
&& let Some((plan, _)) = canonical_boundary_stages(
|
||||
Arc::from(prom_expr.to_string()),
|
||||
Arc::from(expression.as_str()),
|
||||
&tree,
|
||||
&children,
|
||||
&source_tags,
|
||||
@@ -1477,6 +1495,7 @@ impl PromPlanner {
|
||||
format,
|
||||
root_kind,
|
||||
fused_output_tags,
|
||||
expression_field_names,
|
||||
)
|
||||
{
|
||||
return Ok(Some(native_planned(
|
||||
@@ -1510,7 +1529,7 @@ impl PromPlanner {
|
||||
// caller asked for: it names one column per value the partial state accumulates,
|
||||
// while the encoded frame carries those values inside its own layout.
|
||||
let Some(value_schema) =
|
||||
Self::fused_root_schema(&tree, &source_tags, &time, root_kind)
|
||||
Self::fused_root_schema(&expression, &tree, &source_tags, &time, root_kind)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -1525,7 +1544,7 @@ impl PromPlanner {
|
||||
};
|
||||
let publish_grid = Arc::clone(&grid);
|
||||
let node = plan_split_aggregate(
|
||||
Arc::from(prom_expr.to_string()),
|
||||
Arc::from(expression.as_str()),
|
||||
partial,
|
||||
final_tree,
|
||||
FusedSeriesBinding {
|
||||
@@ -1556,7 +1575,7 @@ impl PromPlanner {
|
||||
if required_placement || matches!(locality, InputLocality::SinglePartition) {
|
||||
let segment_metric = if cross_table { None } else { metric.clone() };
|
||||
let plan = segment_tree(
|
||||
Arc::from(prom_expr.to_string()),
|
||||
Arc::from(expression.as_str()),
|
||||
&tree,
|
||||
&children,
|
||||
&source_tags,
|
||||
@@ -1575,7 +1594,8 @@ impl PromPlanner {
|
||||
// that is not a tag, and a partitioned table whose occurrences are source-backed.
|
||||
any_split_series || (per_source_inputs && any_partitioned),
|
||||
fused_output_tags,
|
||||
PromPlanner::fused_root_schema,
|
||||
expression_field_names,
|
||||
PromPlanner::segmented_root_schema,
|
||||
);
|
||||
if let Some(plan) = plan {
|
||||
return Ok(Some(native_planned(
|
||||
@@ -1609,20 +1629,9 @@ impl PromPlanner {
|
||||
let Some(root_fields) = native_root_fields(tree.root()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
// A root field can share a name with a tag or the time index, and under `or` the right
|
||||
// table's tags join the left table's columns. The output names each column once, so such a
|
||||
// statement keeps the existing plan.
|
||||
let mut column_names = std::collections::HashSet::new();
|
||||
if !std::iter::once(&time)
|
||||
.chain(&root_fields)
|
||||
.chain(&output_tags)
|
||||
.all(|name| column_names.insert(name.as_str()))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let output_schema = match format {
|
||||
FusedOutputFormat::PlainRows => {
|
||||
fused_output_schema(&root_fields, &output_tags, &time, root_kind)?
|
||||
fused_output_schema(&expression, &root_fields, &output_tags, &time, root_kind)?
|
||||
}
|
||||
FusedOutputFormat::SeriesFrame => {
|
||||
let frame =
|
||||
@@ -1645,7 +1654,7 @@ impl PromPlanner {
|
||||
// binding is that grid rather than a per-shape choice.
|
||||
let publish_grid = Arc::clone(&grid);
|
||||
let node = FusedSeries::try_new_with_format(
|
||||
Arc::from(prom_expr.to_string()),
|
||||
Arc::from(expression.as_str()),
|
||||
tree,
|
||||
FusedSeriesBinding {
|
||||
grid: publish_grid,
|
||||
@@ -1673,36 +1682,56 @@ impl PromPlanner {
|
||||
|
||||
/// The columns a [`FusedSeries`] node over `tree` produces.
|
||||
///
|
||||
/// Field and tag names are derived from the native tree; a walk that cannot derive a tag set declines
|
||||
/// rather than consulting the legacy planner.
|
||||
/// Fields and tags are derived from the native tree; the expression names the output fields.
|
||||
/// If this walk cannot derive a tag set, the native route declines rather than consulting the
|
||||
/// legacy planner.
|
||||
///
|
||||
/// `None` when the root is not a value the node can publish, or when a field would share a name with a
|
||||
/// tag or the time index.
|
||||
/// `Ok(None)` when the root is not a value the node can publish. A schema the output cannot
|
||||
/// declare is an error rather than a decline: duplicate names violate the expression-named
|
||||
/// native output contract, so construction reports the collision instead of publishing a
|
||||
/// layout the reader cannot address.
|
||||
fn fused_root_schema(
|
||||
expression: &str,
|
||||
tree: &NativeTree,
|
||||
source_tags: &[Vec<String>],
|
||||
time: &str,
|
||||
root_kind: FieldKind,
|
||||
) -> Result<Option<DFSchemaRef>> {
|
||||
let output_tags = match &tree.root().output {
|
||||
NativeType::Scalar(_) => Vec::new(),
|
||||
NativeType::Samples(_) => {
|
||||
let Some(tags) = fused_output_tags(tree.root(), source_tags) else {
|
||||
return Ok(None);
|
||||
};
|
||||
with_row_name_column(tree.root(), tags)
|
||||
}
|
||||
NativeType::AggregateState(_) => return Ok(None),
|
||||
};
|
||||
let Some(root_fields) = native_root_fields(tree.root()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
// The kind is the one the statement's own root answers, derived before segmentation cut
|
||||
// anything. It cannot be recomputed here: a cut prefix is replaced by an input leaf whose
|
||||
// samples are already evaluated, so walking this tree would read every such leaf as the
|
||||
// kind its scan carried. `count_over_time(h[5m]) + on (__name__) ...` answers floats, and
|
||||
// its replaced leaf carries floats, while the scan below was a histogram.
|
||||
fused_output_schema(expression, &root_fields, &output_tags, time, root_kind).map(Some)
|
||||
}
|
||||
|
||||
/// The rows layout a segmented statement root publishes, under the segment builders' callback.
|
||||
///
|
||||
/// A segment cannot carry this crate's error type, so a layout it cannot declare is a decline
|
||||
/// here: the route keeps the plan the existing planner would have built for the same statement.
|
||||
fn segmented_root_schema(
|
||||
expression: &str,
|
||||
tree: &NativeTree,
|
||||
source_tags: &[Vec<String>],
|
||||
time: &str,
|
||||
root_kind: FieldKind,
|
||||
) -> Option<DFSchemaRef> {
|
||||
let output_tags = match &tree.root().output {
|
||||
NativeType::Scalar(_) => Vec::new(),
|
||||
NativeType::Samples(_) => {
|
||||
with_row_name_column(tree.root(), fused_output_tags(tree.root(), source_tags)?)
|
||||
}
|
||||
NativeType::AggregateState(_) => return None,
|
||||
};
|
||||
let root_fields = native_root_fields(tree.root())?;
|
||||
let mut column_names = std::collections::HashSet::new();
|
||||
let columns = std::iter::once(time)
|
||||
.chain(root_fields.iter().map(String::as_str))
|
||||
.chain(output_tags.iter().map(String::as_str));
|
||||
if !columns.clone().all(|name| column_names.insert(name)) {
|
||||
return None;
|
||||
}
|
||||
// The kind is the one the statement's own root answers, derived before segmentation cut anything.
|
||||
// It cannot be recomputed here: a cut prefix is replaced by an input leaf whose samples are
|
||||
// already evaluated.
|
||||
fused_output_schema(&root_fields, &output_tags, time, root_kind).ok()
|
||||
Self::fused_root_schema(expression, tree, source_tags, time, root_kind)
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// The statement being planned, rebuilt from the evaluation numbers the planner holds.
|
||||
@@ -1821,18 +1850,67 @@ fn native_root_fields(node: &NativeNode) -> Option<Vec<String>> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The output schema of a fused node: its time index, one column per field, then its tags.
|
||||
/// The names an expression gives the value columns of `fields`.
|
||||
///
|
||||
/// The columns themselves are named by [`segment::fused_output_schema`], which is where a segment
|
||||
/// asks for the same layout; this is that answer under this crate's error type, which is what the
|
||||
/// planner's own call sites report.
|
||||
/// One value is named after the whole expression; several append their compiled field names to it,
|
||||
/// because one name cannot stand for more than one column. Only the published layout is named this
|
||||
/// way: the fields inside the native tree keep the names the compiler gave them.
|
||||
fn expression_field_names(expression: &str, fields: &[String]) -> Vec<String> {
|
||||
fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
if fields.len() == 1 {
|
||||
expression.to_string()
|
||||
} else {
|
||||
format!("{expression}.{field}")
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The columns a [`FusedSeries`] node produces: its time index, one column per value, then its tags.
|
||||
///
|
||||
/// A single value is named after the expression, and multiple values append their compiled field
|
||||
/// names to it, so a value is never published under a name the reader cannot address. The names
|
||||
/// inside the native tree are unchanged.
|
||||
///
|
||||
/// The layout itself is [`segment::fused_output_schema`]'s, which is where a segment asks for the
|
||||
/// same columns; this is that answer under this crate's error type, which is what the planner's own
|
||||
/// call sites report.
|
||||
fn fused_output_schema(
|
||||
expression: &str,
|
||||
fields: &[String],
|
||||
tags: &[String],
|
||||
time_index: &str,
|
||||
field_kind: FieldKind,
|
||||
) -> Result<DFSchemaRef> {
|
||||
segment::fused_output_schema(fields, tags, time_index, field_kind)
|
||||
let value_names = expression_field_names(expression, fields);
|
||||
// A name appears twice when a value named after its expression - a bare selector is named after
|
||||
// its metric - lands on the table's own time index or on one of its tags, or when two values
|
||||
// take one name. `DFSchema` rejects that, but reports only that building the plan failed, which
|
||||
// names no column; the collision is detected here so the error names the columns that collide.
|
||||
let mut names = Vec::with_capacity(value_names.len() + tags.len() + 1);
|
||||
names.push(time_index.to_string());
|
||||
names.extend(value_names.iter().cloned());
|
||||
names.extend(tags.iter().cloned());
|
||||
let mut occurrences: std::collections::HashMap<&str, usize> =
|
||||
std::collections::HashMap::with_capacity(names.len());
|
||||
for name in &names {
|
||||
*occurrences.entry(name.as_str()).or_default() += 1;
|
||||
}
|
||||
let mut reported = std::collections::HashSet::new();
|
||||
let duplicates = names
|
||||
.iter()
|
||||
.filter(|name| occurrences[name.as_str()] > 1 && reported.insert(name.as_str()))
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(
|
||||
duplicates.is_empty(),
|
||||
DuplicateOutputColumnsSnafu {
|
||||
names: duplicates.join(", ")
|
||||
}
|
||||
);
|
||||
segment::fused_output_schema(&value_names, tags, time_index, field_kind)
|
||||
.context(DataFusionPlanningSnafu)
|
||||
}
|
||||
|
||||
@@ -2911,8 +2989,10 @@ mod tests {
|
||||
("x or y", two()),
|
||||
] {
|
||||
let tree = compiled_test_tree(query);
|
||||
let schema = PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float)
|
||||
.unwrap_or_else(|| panic!("{query}: a samples root declares a rows layout"));
|
||||
let schema =
|
||||
PromPlanner::fused_root_schema(query, &tree, &sources, "ts", FieldKind::Float)
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| panic!("{query}: a samples root declares a rows layout"));
|
||||
let names = schema
|
||||
.fields()
|
||||
.iter()
|
||||
@@ -2932,9 +3012,13 @@ mod tests {
|
||||
assert_eq!(ArrowDataType::Utf8, *name.data_type(), "{query}");
|
||||
}
|
||||
let scalar = compiled_test_tree("1 + 2");
|
||||
let schema = PromPlanner::fused_root_schema(&scalar, &[], "ts", FieldKind::Float).unwrap();
|
||||
let schema = PromPlanner::fused_root_schema("1 + 2", &scalar, &[], "ts", FieldKind::Float)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
vec!["ts", "value"],
|
||||
// A scalar root declares no name column, and its one value is named after the
|
||||
// expression, as every other rows layout names its values.
|
||||
vec!["ts", "1 + 2"],
|
||||
schema
|
||||
.fields()
|
||||
.iter()
|
||||
@@ -2948,10 +3032,11 @@ mod tests {
|
||||
fn a_present_name_tag_is_not_declared_twice() {
|
||||
let sources = [vec!["host".to_string(), METRIC_NAME.to_string()]];
|
||||
let tree = compiled_test_tree("x");
|
||||
let schema =
|
||||
PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float).unwrap();
|
||||
let schema = PromPlanner::fused_root_schema("x", &tree, &sources, "ts", FieldKind::Float)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
vec!["ts", "v0", "host", METRIC_NAME],
|
||||
vec!["ts", "x", "host", METRIC_NAME],
|
||||
schema
|
||||
.fields()
|
||||
.iter()
|
||||
@@ -2960,19 +3045,34 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A value column that is itself named `__name__` is a column of the result, not the name of a
|
||||
/// row: the layout names each column once, so such a statement keeps the existing plan.
|
||||
/// The value columns are named after the expression, so a tree whose own field is named
|
||||
/// `__name__` still publishes that value under the expression's name - the name column is the
|
||||
/// layout's own. A time index of that name collides with it, and the collision is reported
|
||||
/// rather than published as a layout the reader cannot address.
|
||||
#[test]
|
||||
fn a_column_named_after_the_metric_name_declines_the_rows_layout() {
|
||||
fn a_time_index_named_after_the_metric_name_is_reported() {
|
||||
let sources = [vec!["host".to_string()]];
|
||||
// The field is renamed by the expression, so it no longer stands where the name column is.
|
||||
let tree = compiled_test_tree_with_fields("x", &[METRIC_NAME]);
|
||||
assert!(PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float).is_none());
|
||||
// A time index of that name collides the same way.
|
||||
let tree = compiled_test_tree("x");
|
||||
assert!(
|
||||
PromPlanner::fused_root_schema(&tree, &sources, METRIC_NAME, FieldKind::Float)
|
||||
.is_none()
|
||||
let schema = PromPlanner::fused_root_schema("x", &tree, &sources, "ts", FieldKind::Float)
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| panic!("the value is named after the expression"));
|
||||
assert_eq!(
|
||||
vec!["ts", "x", "host", METRIC_NAME],
|
||||
schema
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| field.name().as_str())
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
// A time index of that name collides with the row-name column, and the error names it.
|
||||
let tree = compiled_test_tree("x");
|
||||
let error =
|
||||
PromPlanner::fused_root_schema("x", &tree, &sources, METRIC_NAME, FieldKind::Float)
|
||||
.expect_err("a time index named after the metric collides with the name column");
|
||||
let message = error.to_string().to_lowercase();
|
||||
assert!(message.contains("duplicate"), "{message}");
|
||||
assert!(message.contains(METRIC_NAME), "{message}");
|
||||
}
|
||||
|
||||
/// A `__name__` rewrite publishes a name column, canonicalizes the complete runtime label set,
|
||||
@@ -3145,7 +3245,9 @@ mod tests {
|
||||
.map(|field| field.name.as_ref())
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
let expression = "sum by (zone, host) ((x))";
|
||||
let output = fused_output_schema(
|
||||
expression,
|
||||
&["v0".to_string(), "v1".to_string()],
|
||||
&["zone".to_string(), "host".to_string()],
|
||||
"ts",
|
||||
@@ -3153,7 +3255,13 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
vec!["ts", "v0", "v1", "zone", "host"],
|
||||
vec![
|
||||
"ts".to_string(),
|
||||
format!("{expression}.v0"),
|
||||
format!("{expression}.v1"),
|
||||
"zone".to_string(),
|
||||
"host".to_string(),
|
||||
],
|
||||
output
|
||||
.fields()
|
||||
.iter()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -87,6 +87,7 @@ use promql::vector_match::MatchingLabels;
|
||||
use query::datafusion::QUERY_PARALLELISM_HINT;
|
||||
use query::dist_plan::MergeScanExec;
|
||||
use query::options::QueryOptions;
|
||||
use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
|
||||
use query::physical_wrapper::{PhysicalPlanWrapper, PhysicalPlanWrapperRef};
|
||||
use servers::query_handler::sql::SqlQueryHandler;
|
||||
use session::context::{QueryContext, QueryContextRef};
|
||||
@@ -430,9 +431,6 @@ struct AggregateCase {
|
||||
partial_state: Option<&'static str>,
|
||||
/// Whether this case's global aggregate has an explicitly specified single owner.
|
||||
single_owner: bool,
|
||||
/// Native output schema. The native aggregate preserves its source field name (`v`), rather
|
||||
/// than the legacy planner's computed-expression name.
|
||||
native_value_column: &'static str,
|
||||
}
|
||||
|
||||
/// Builds one case from a PromQL expression and its `(labels, value)` rows.
|
||||
@@ -454,7 +452,6 @@ fn case(promql: String, expected: &[(&[(&str, &str)], f64)], per_partition: bool
|
||||
root_values: None,
|
||||
partial_state: None,
|
||||
single_owner: false,
|
||||
native_value_column: "v",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -485,6 +482,19 @@ fn tql_eval(promql: &str) -> String {
|
||||
format!("TQL EVAL ({EVAL_SECS}, {EVAL_SECS}, '1m') {promql}")
|
||||
}
|
||||
|
||||
fn expression_name(promql: &str) -> String {
|
||||
let query = PromQuery {
|
||||
query: promql.to_string(),
|
||||
..PromQuery::default()
|
||||
};
|
||||
let QueryStatement::Promql(stmt, _) =
|
||||
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
stmt.expr.to_string()
|
||||
}
|
||||
|
||||
/// Runs `promql` at [`EVAL_SECS`] and returns its rows with the native nodes its plans held.
|
||||
async fn run_query(
|
||||
instance: &Arc<Instance>,
|
||||
@@ -1192,12 +1202,12 @@ fn assert_rows(actual: &[ResultRow], expected: &[ResultRow], context: &str) {
|
||||
}
|
||||
|
||||
/// Asserts the native boundary's ordered names and types, independently of legacy names.
|
||||
fn assert_native_column_layout(native: &QueryResult, tags: &[&str], query: &str) {
|
||||
let mut expected = vec!["ts", "v"];
|
||||
expected.extend_from_slice(tags);
|
||||
fn assert_native_column_layout(native: &QueryResult, tags: &[&str], promql: &str, query: &str) {
|
||||
let mut expected = vec!["ts".to_string(), expression_name(promql)];
|
||||
expected.extend(tags.iter().map(|tag| (*tag).to_string()));
|
||||
// The rows layout declares the per-row name column after the labels; an aggregate computes a
|
||||
// value of its own, so that column is null here.
|
||||
expected.push("__name__");
|
||||
expected.push("__name__".to_string());
|
||||
assert_eq!(
|
||||
expected, native.columns,
|
||||
"`{query}`: native boundary columns"
|
||||
@@ -1281,7 +1291,7 @@ fn assert_native_columns(
|
||||
);
|
||||
assert_eq!(
|
||||
expected_value_column, native.value_column,
|
||||
"`{query}`: native value column must use the compiled root field name"
|
||||
"`{query}`: native value column must use the statement expression"
|
||||
);
|
||||
assert_eq!(
|
||||
1,
|
||||
@@ -1774,7 +1784,7 @@ async fn check_two_source_segregation(
|
||||
legacy.columns,
|
||||
"`{query}`: the legacy plan names the value column after its expression"
|
||||
);
|
||||
assert_native_column_layout(&native, &[], &query);
|
||||
assert_native_column_layout(&native, &[], promql, &query);
|
||||
assert_rows(
|
||||
&native.rows,
|
||||
&legacy.rows,
|
||||
@@ -1883,7 +1893,7 @@ async fn check_nested_aggregate_case(
|
||||
assert_rows(&legacy.rows, &case.expected, &format!("legacy `{query}`"));
|
||||
|
||||
let (native, capture) = run_query(instance, &case.promql, true, observer).await;
|
||||
assert_native_column_layout(&native, &[], &query);
|
||||
assert_native_column_layout(&native, &[], &case.promql, &query);
|
||||
assert_rows(
|
||||
&native.rows,
|
||||
&legacy.rows,
|
||||
@@ -1916,7 +1926,7 @@ async fn check_case(instance: &Arc<Instance>, observer: &NativePlanObserver, cas
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>();
|
||||
assert_native_column_layout(&native, &tags, &query);
|
||||
assert_native_column_layout(&native, &tags, promql, &query);
|
||||
if promql.starts_with("count ") {
|
||||
assert!(
|
||||
legacy.column_types.contains(&DataType::Int64),
|
||||
@@ -1928,8 +1938,8 @@ async fn check_case(instance: &Arc<Instance>, observer: &NativePlanObserver, cas
|
||||
&legacy.rows,
|
||||
&format!("native `{query}` vs legacy"),
|
||||
);
|
||||
assert_native_schema(&native, case.native_value_column, &case.expected, &query);
|
||||
assert_native_columns(&native, &legacy, &query, "v");
|
||||
assert_native_schema(&native, &expression_name(promql), &case.expected, &query);
|
||||
assert_native_columns(&native, &legacy, &query, &expression_name(promql));
|
||||
assert_rows(&native.rows, &case.expected, &format!("native `{query}`"));
|
||||
|
||||
assert_eq!(
|
||||
@@ -2190,7 +2200,7 @@ async fn native_aggregate_topk_over_global_partitions() {
|
||||
let (legacy, legacy_capture) = run(false).await;
|
||||
assert!(legacy_capture.fused.is_empty() && legacy_capture.finalize.is_empty());
|
||||
let (native, capture) = run(true).await;
|
||||
assert_native_columns(&native, &legacy, &sql, "v");
|
||||
assert_native_columns(&native, &legacy, &sql, &expression_name(&promql));
|
||||
assert_rows(
|
||||
&native.rows,
|
||||
&legacy.rows,
|
||||
@@ -2225,7 +2235,7 @@ async fn native_aggregate_topk_over_global_partitions() {
|
||||
}
|
||||
}
|
||||
expected.sort_by(|a, b| (a.ts_millis, &a.labels).cmp(&(b.ts_millis, &b.labels)));
|
||||
assert_native_schema(&native, "v", &expected, &sql);
|
||||
assert_native_schema(&native, &expression_name(&promql), &expected, &sql);
|
||||
assert_rows(&native.rows, &expected, &format!("expected `{sql}`"));
|
||||
// `topk` above a global aggregate is not split, so the scan side folds per series and the
|
||||
// ranking runs above it. A plan may hold several fused nodes (the frontend root and the region
|
||||
@@ -2295,9 +2305,13 @@ async fn native_aggregate_clamp_executes_on_both_routes() {
|
||||
close_enough(native.rows[0].value.unwrap(), 10.0),
|
||||
"native `{query}` value"
|
||||
);
|
||||
assert_eq!("v", native.value_column, "native `{query}` value column");
|
||||
// The rows layout: the time index, the one value, and the declared name column, which an
|
||||
// aggregate leaves null.
|
||||
assert_eq!(
|
||||
expression_name(&promql),
|
||||
native.value_column,
|
||||
"native `{query}` value column"
|
||||
);
|
||||
// The rows layout: the time index, the one expression-named value, and the declared name
|
||||
// column, which an aggregate leaves null.
|
||||
assert_eq!(3, native.columns.len(), "native `{query}` columns");
|
||||
assert_eq!("__name__", native.columns[2], "native `{query}` columns");
|
||||
|
||||
@@ -2781,9 +2795,9 @@ async fn native_histogram_over_the_cut_splits_and_matches_legacy() {
|
||||
// reports over the complete input, labels included - is compared before the plan shape, so a
|
||||
// shape regression cannot hide it.
|
||||
let tags: &[&str] = if *single_owner { &[] } else { &["job"] };
|
||||
assert_native_column_layout(&native, tags, &query);
|
||||
assert_native_schema(&native, "v", &legacy.rows, &query);
|
||||
assert_native_columns(&native, &legacy, &query, "v");
|
||||
assert_native_column_layout(&native, tags, promql, &query);
|
||||
assert_native_schema(&native, &expression_name(promql), &legacy.rows, &query);
|
||||
assert_native_columns(&native, &legacy, &query, &expression_name(promql));
|
||||
assert_rows(
|
||||
&native.rows,
|
||||
&legacy.rows,
|
||||
@@ -3032,15 +3046,19 @@ async fn native_bare_selector_aggregate_value_columns() {
|
||||
let mark = observer.mark();
|
||||
let (native_names, native_types, native_rows, capture) =
|
||||
read(&instance, &observer, &promql, true).await;
|
||||
let expression = expression_name(&promql);
|
||||
let mut expected_names = vec!["ts".to_string()];
|
||||
match expected_values {
|
||||
1 => expected_names.push(expression),
|
||||
2 => expected_names.extend([format!("{expression}.a"), format!("{expression}.b")]),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
if !single_owner {
|
||||
expected_names.push("job".to_string());
|
||||
}
|
||||
// The rows layout declares the per-row name column after the labels; every statement in
|
||||
// this case is an aggregate, which computes its value and leaves that column null.
|
||||
let expected_names = match (expected_values, single_owner) {
|
||||
(1, true) => vec!["ts", "v", "__name__"],
|
||||
(1, false) => vec!["ts", "v", "job", "__name__"],
|
||||
(2, true) => vec!["ts", "a", "b", "__name__"],
|
||||
(2, false) => vec!["ts", "a", "b", "job", "__name__"],
|
||||
_ => unreachable!(),
|
||||
};
|
||||
expected_names.push("__name__".to_string());
|
||||
assert_eq!(
|
||||
expected_names, native_names,
|
||||
"`{query}` native boundary columns"
|
||||
|
||||
@@ -5256,11 +5256,15 @@ async fn distributed_native_promql_zero_match_or_real() {
|
||||
);
|
||||
|
||||
let (columns, rows) = native_labelled_rows(output).await;
|
||||
assert_eq!(
|
||||
columns,
|
||||
vec!["__name__", "host", "ts", "val"],
|
||||
"{expression}"
|
||||
);
|
||||
// The value is named after the statement's expression, not `val`.
|
||||
let mut expected_columns = vec![
|
||||
"__name__".to_string(),
|
||||
"host".to_string(),
|
||||
"ts".to_string(),
|
||||
expression_name(expression),
|
||||
];
|
||||
expected_columns.sort();
|
||||
assert_eq!(columns, expected_columns, "{expression}");
|
||||
let actual = rows
|
||||
.iter()
|
||||
.map(|(time, labels, value)| {
|
||||
@@ -5454,7 +5458,7 @@ async fn distributed_native_promql_metric_discovery_reconciles_mixed_ordering()
|
||||
.clone()
|
||||
});
|
||||
let values = batch
|
||||
.column_by_name("val")
|
||||
.column_by_name(&expression_name(expression))
|
||||
.expect("one value column")
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
@@ -5626,9 +5630,7 @@ async fn data_free_rows(output: Output) -> (Vec<String>, Vec<(i64, f64)>) {
|
||||
(columns, rows)
|
||||
}
|
||||
|
||||
/// A statement that reads no table plans natively over the wire, where there is no region to place
|
||||
/// anything in: the frontend evaluates one fused node, so the answer keeps the native column name
|
||||
/// instead of the one the existing planner derives from an expression.
|
||||
/// A statement that reads no table plans natively over the wire, with an expression-named value.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn distributed_native_promql_data_free_statement_plans_natively() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
@@ -5644,15 +5646,15 @@ async fn distributed_native_promql_data_free_statement_plans_natively() {
|
||||
.build(false)
|
||||
.await;
|
||||
let instance = cluster.fe_instance().clone();
|
||||
for (expression, expected_columns, expected_rows) in [
|
||||
for (expression, labels, expected_rows) in [
|
||||
// `start`/`end` are seconds, so this one-step range evaluates at 3000ms.
|
||||
("time()", &["time", "value"][..], &[(3000, 3.0)][..]),
|
||||
("time()", &[][..], &[(3000, 3.0)][..]),
|
||||
(
|
||||
// A samples root declares the name of each row it publishes, and `vector(1)` has
|
||||
// none, so the column is present and null. `time()` above is a scalar root and keeps
|
||||
// two columns.
|
||||
r#"label_replace(vector(1), "host", "host1", "", "")"#,
|
||||
&["time", "value", "host", "__name__"][..],
|
||||
&["host", "__name__"][..],
|
||||
&[(3000, 1.0)][..],
|
||||
),
|
||||
] {
|
||||
@@ -5679,11 +5681,17 @@ async fn distributed_native_promql_data_free_statement_plans_natively() {
|
||||
"a data-free statement reads no region: {plan:?}"
|
||||
);
|
||||
let (columns, rows) = data_free_rows(output).await;
|
||||
let QueryStatement::Promql(stmt, _) =
|
||||
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let mut expected_columns = vec!["time".to_string(), stmt.expr.to_string()];
|
||||
expected_columns.extend(labels.iter().map(|label| (*label).to_string()));
|
||||
assert_eq!(expected_columns, columns, "{expression}");
|
||||
assert_eq!(expected_rows, rows.as_slice(), "{expression}");
|
||||
|
||||
// The same statement through the existing planner answers the same rows; only the native
|
||||
// route names the value column after the tree rather than after an expression.
|
||||
// The same statement through the existing planner answers the same rows.
|
||||
let legacy = instance
|
||||
.do_promql_query(&query, native_query_context(false, 2))
|
||||
.await
|
||||
@@ -6805,9 +6813,20 @@ async fn native_promql_sort_absent_and_histogram_quantile_match_legacy() {
|
||||
let legacy_rows = legacy.iter().map(|batch| batch.num_rows()).sum::<usize>();
|
||||
assert_eq!(native_rows, legacy_rows);
|
||||
assert!(native_rows >= 1, "histogram_quantile must produce data");
|
||||
// The rows root lays its columns out as time, values, labels, and the name last.
|
||||
let query = PromQuery {
|
||||
query: shape.to_string(),
|
||||
..PromQuery::default()
|
||||
};
|
||||
let QueryStatement::Promql(stmt, _) =
|
||||
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let expression = stmt.expr.to_string();
|
||||
// The rows root lays its columns out as time, the expression-named value, the labels,
|
||||
// and the name last.
|
||||
assert_eq!(
|
||||
vec!["ts", "val", "host", "__name__"],
|
||||
vec!["ts", expression.as_str(), "host", "__name__"],
|
||||
native_columns
|
||||
.iter()
|
||||
.map(|column| column.name.as_str())
|
||||
@@ -6902,15 +6921,132 @@ async fn native_promql_sort_absent_and_histogram_quantile_match_legacy() {
|
||||
// keep the metric name; `absent` answers a value of its own for a label set it derives
|
||||
// from the matcher, and Prometheus' `absent` leaves the name out of that set. The
|
||||
// existing route has no name column at all, so the column is checked against what the
|
||||
// shape answers and left out of the comparison.
|
||||
// shape answers and left out of the comparison below.
|
||||
//
|
||||
// Both routes answer the same rows; only the one value column's header differs,
|
||||
// because the existing planner keeps the tree's field name while the native route
|
||||
// names that column after the expression. Compare what the columns carry - the
|
||||
// timestamp, the label set and the value, in response order - and the native naming.
|
||||
let reported = if shape.starts_with("absent") {
|
||||
ReportedName::Nothing
|
||||
} else {
|
||||
ReportedName::Metric("native_shapes")
|
||||
};
|
||||
let native = named_rows_text(native, reported).await;
|
||||
let legacy = named_rows_text(legacy, ReportedName::NoColumn).await;
|
||||
assert_eq!(native, legacy, "{shape}; SQL: {sql}");
|
||||
let collect = |output: Output| async move {
|
||||
match output.data {
|
||||
OutputData::Stream(stream) => collect_batches(stream).await.unwrap(),
|
||||
OutputData::RecordBatches(batches) => batches,
|
||||
_ => unreachable!("query returns record batches"),
|
||||
}
|
||||
};
|
||||
let rows = |batches: &common_recordbatch::RecordBatches, reported: ReportedName<'_>| {
|
||||
batches
|
||||
.iter()
|
||||
.flat_map(|batch| {
|
||||
let batch = batch.df_record_batch();
|
||||
let schema = batch.schema();
|
||||
// The name column is checked against what the shape answers and kept out of
|
||||
// the label set, so the rows of the two routes compare whole.
|
||||
checked_name_column(schema.fields(), batch.columns(), reported);
|
||||
let value_index = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.position(|field| field.data_type() == &DataType::Float64)
|
||||
.expect("one float value column");
|
||||
let timestamps = batch
|
||||
.column_by_name("ts")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<TimestampMillisecondArray>()
|
||||
.unwrap();
|
||||
let labels = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|field| {
|
||||
!matches!(
|
||||
field.data_type(),
|
||||
DataType::Float64 | DataType::Timestamp(..)
|
||||
) && field.name() != "__name__"
|
||||
})
|
||||
.map(|field| {
|
||||
let column = cast(
|
||||
batch.column_by_name(field.name()).unwrap().as_ref(),
|
||||
&DataType::Utf8,
|
||||
)
|
||||
.unwrap();
|
||||
(
|
||||
field.name().clone(),
|
||||
column
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let values =
|
||||
cast(batch.column(value_index).as_ref(), &DataType::Float64).unwrap();
|
||||
let values = values.as_any().downcast_ref::<Float64Array>().unwrap();
|
||||
(0..batch.num_rows())
|
||||
.map(|row| {
|
||||
(
|
||||
timestamps.value(row),
|
||||
labels
|
||||
.iter()
|
||||
.filter(|(_, column)| {
|
||||
column.is_valid(row) && !column.value(row).is_empty()
|
||||
})
|
||||
.map(|(name, column)| {
|
||||
(name.clone(), column.value(row).to_string())
|
||||
})
|
||||
.collect::<std::collections::BTreeMap<_, _>>(),
|
||||
values.value(row),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let native = collect(native).await;
|
||||
let legacy = collect(legacy).await;
|
||||
let native_names = native
|
||||
.schema()
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.map(|column| column.name.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let native_rows = rows(&native, reported);
|
||||
let legacy_rows = rows(&legacy, ReportedName::NoColumn);
|
||||
assert_eq!(native_rows, legacy_rows, "{shape}; SQL: {sql}");
|
||||
if shape.starts_with("sort") {
|
||||
assert!(!native_rows.is_empty(), "{shape} must produce data: {sql}");
|
||||
}
|
||||
let legacy_labels = legacy
|
||||
.schema()
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.filter(|column| {
|
||||
!column.data_type.is_timestamp()
|
||||
&& column.data_type
|
||||
!= datatypes::prelude::ConcreteDataType::float64_datatype()
|
||||
})
|
||||
.map(|column| column.name.clone())
|
||||
.collect::<Vec<_>>();
|
||||
// The native answer's columns are the time index, the expression-named value, the
|
||||
// labels the existing route also declares, and the name column it appends last.
|
||||
let mut expected_names = vec!["ts".to_string(), expression_name(shape)];
|
||||
expected_names.extend(legacy_labels);
|
||||
expected_names.push("__name__".to_string());
|
||||
assert_eq!(
|
||||
native_names, expected_names,
|
||||
"{shape}: native boundary columns"
|
||||
);
|
||||
assert!(
|
||||
native_names[2..native_names.len() - 1]
|
||||
.iter()
|
||||
.all(|name| matches!(name.as_str(), "host" | "le")),
|
||||
"{shape}: native boundary columns: {native_names:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7541,11 +7677,10 @@ async fn native_kind_rows(output: Output) -> Vec<(i64, String, String, f64)> {
|
||||
let hosts = hosts.as_any().downcast_ref::<StringArray>().unwrap();
|
||||
let kinds = kinds.as_any().downcast_ref::<StringArray>().unwrap();
|
||||
let values = batch
|
||||
.column_by_name("val")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.unwrap();
|
||||
.columns()
|
||||
.iter()
|
||||
.find_map(|column| column.as_any().downcast_ref::<Float64Array>())
|
||||
.expect("one value column");
|
||||
for row in 0..batch.num_rows() {
|
||||
rows.push((
|
||||
timestamps.value(row),
|
||||
@@ -8157,8 +8292,15 @@ async fn native_promql_zero_match_or_real_tql() {
|
||||
// A rows answer reports the statement's own columns, the per-row name column among them.
|
||||
// `alpha_total` reads one named metric, and `or` reports each side with the label set it
|
||||
// read, so every row here is a series of that metric and keeps its name while the empty
|
||||
// side adds nothing.
|
||||
assert_eq!(columns, vec!["__name__", "host", "ts", "val"], "{query}");
|
||||
// side adds nothing. The value is named after the statement's expression, not `val`.
|
||||
let mut expected_columns = vec![
|
||||
"__name__".to_string(),
|
||||
"host".to_string(),
|
||||
"ts".to_string(),
|
||||
expression_name(query),
|
||||
];
|
||||
expected_columns.sort();
|
||||
assert_eq!(columns, expected_columns, "{query}");
|
||||
let actual = rows
|
||||
.iter()
|
||||
.map(|(time, labels, value)| {
|
||||
@@ -8499,6 +8641,19 @@ async fn discovery_query(instance: &Arc<Instance>, promql: &str) -> Output {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn expression_name(promql: &str) -> String {
|
||||
let query = PromQuery {
|
||||
query: promql.to_string(),
|
||||
..PromQuery::default()
|
||||
};
|
||||
let QueryStatement::Promql(stmt, _) =
|
||||
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
stmt.expr.to_string()
|
||||
}
|
||||
|
||||
/// `(time, labels, value)` of every row, sorted, whatever the columns are named: the time index is
|
||||
/// the timestamp column, the value the one `Float64` column, and every other column a label that a
|
||||
/// NULL or empty value leaves out. Also returns the column names.
|
||||
@@ -8860,7 +9015,18 @@ async fn native_promql_cross_table_pairs_over_different_schemas() {
|
||||
2,
|
||||
"{context}: expected one distinct SeriesScanExec per selector occurrence"
|
||||
);
|
||||
assert_eq!(native_columns, columns, "{context}");
|
||||
let mut expected_columns = columns
|
||||
.into_iter()
|
||||
.map(|column| {
|
||||
if column == "val" {
|
||||
expression_name(query)
|
||||
} else {
|
||||
column.to_string()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
expected_columns.sort();
|
||||
assert_eq!(native_columns, expected_columns, "{context}");
|
||||
assert_eq!(native.len(), expected.len(), "{context}: {native:?}");
|
||||
for (actual, expected) in native.iter().zip(&expected) {
|
||||
assert_eq!(
|
||||
@@ -9055,7 +9221,27 @@ async fn native_promql_three_selector_trees_bind_every_occurrence() {
|
||||
.map(|field| field.name().clone())
|
||||
.collect::<Vec<_>>();
|
||||
plan_columns.sort();
|
||||
assert_eq!(plan_columns, columns, "{query}");
|
||||
let expression_query = PromQuery {
|
||||
query: query.to_string(),
|
||||
..PromQuery::default()
|
||||
};
|
||||
let QueryStatement::Promql(stmt, _) =
|
||||
QueryLanguageParser::parse_promql(&expression_query, &QueryContext::arc()).unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let mut expected_columns = columns
|
||||
.into_iter()
|
||||
.map(|column| {
|
||||
if column == "val" {
|
||||
stmt.expr.to_string()
|
||||
} else {
|
||||
column.to_string()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
expected_columns.sort();
|
||||
assert_eq!(plan_columns, expected_columns, "{query}");
|
||||
assert_eq!(native, expected, "{query}");
|
||||
let legacy = native_result_rows(native_tql_output(&instance, &sql, false).await).await;
|
||||
assert_eq!(native, legacy, "legacy: {query}");
|
||||
@@ -9278,7 +9464,18 @@ async fn native_promql_info_enriches_from_the_info_table() {
|
||||
assert_eq!(plan.name(), "FusedSeriesExec", "{query}");
|
||||
assert_eq!(plan.children().len(), 2, "{query}");
|
||||
assert_eq!(observer.take_raw_sources().len(), 2, "{query}");
|
||||
assert_eq!(native_columns, columns, "{query}");
|
||||
let mut expected_columns = columns
|
||||
.into_iter()
|
||||
.map(|column| {
|
||||
if column == "val" {
|
||||
expression_name(query)
|
||||
} else {
|
||||
column.to_string()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
expected_columns.sort();
|
||||
assert_eq!(native_columns, expected_columns, "{query}");
|
||||
assert_eq!(native, expected, "{query}");
|
||||
|
||||
// The existing plan refuses `info`: "Unsupported expr type: info" for one argument, and
|
||||
@@ -10085,10 +10282,18 @@ async fn native_hybrid_binary_empty_on_grouped_matching_and_errors() {
|
||||
native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
|
||||
let plan = observer.two_input_fused();
|
||||
assert_eq!(plan.name(), "FusedSeriesExec", "{query}");
|
||||
assert_eq!(
|
||||
columns,
|
||||
vec!["__name__", "host", "kind", "ts", "val", "zone_name"]
|
||||
);
|
||||
// The value is named after the statement's expression, and the rows answer reports the
|
||||
// per-row name column among its labels; a math kernel drops the name, so it holds no value.
|
||||
let mut expected_columns = vec![
|
||||
"__name__".to_string(),
|
||||
"host".to_string(),
|
||||
"kind".to_string(),
|
||||
"ts".to_string(),
|
||||
expression_name(query),
|
||||
"zone_name".to_string(),
|
||||
];
|
||||
expected_columns.sort();
|
||||
assert_eq!(columns, expected_columns);
|
||||
assert_eq!(rows, expected, "{query}");
|
||||
let (_, legacy) =
|
||||
native_labelled_rows(native_tql_output(&instance, &sql, false).await).await;
|
||||
@@ -10582,6 +10787,26 @@ async fn native_promql_metric_two_aggregate_pair_matches_legacy() {
|
||||
.iter()
|
||||
.map(|column| column.name.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
let expression_query = PromQuery {
|
||||
query: query.to_string(),
|
||||
..PromQuery::default()
|
||||
};
|
||||
let QueryStatement::Promql(stmt, _) =
|
||||
QueryLanguageParser::parse_promql(&expression_query, &QueryContext::arc()).unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let expression = stmt.expr.to_string();
|
||||
let expected_columns = expected_columns
|
||||
.into_iter()
|
||||
.map(|column| {
|
||||
if column == "val" {
|
||||
expression.as_str()
|
||||
} else {
|
||||
column
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
expected_columns, boundary_columns,
|
||||
"{query}: native boundary columns"
|
||||
@@ -10657,15 +10882,13 @@ async fn native_promql_metric_two_aggregate_pair_matches_legacy() {
|
||||
|
||||
// The columns both routes report: the time index, one value column, and one column per
|
||||
// label. The labels themselves are compared row by row above, and the time index is `ts` on
|
||||
// both. The value column's *name* is not equal, and that is this route's existing naming for
|
||||
// a computed value column rather than anything this shape introduces: the node names its
|
||||
// fields after the tree's own root schema, while the existing planner names a computed column
|
||||
// after its expression (the same difference is pinned for a cross-table pair by
|
||||
// `native_fused_cross_table_pair_reads_each_table_through_its_own_source`). So the
|
||||
// shape is asserted, and the value column's name is pinned as the raw field it comes from.
|
||||
// The rows layout declares the per-row name column and this statement has none to report:
|
||||
// a division computes a value of its own and its inputs are aggregates, so every row's name
|
||||
// is null. The column is therefore the only difference from the existing route's schema.
|
||||
// both. Compare time and label columns independently: the native boundary names that value
|
||||
// column after the canonical expression, where the existing planner names a computed column
|
||||
// after its expression too, and the two texts differ in the spaces a PromQL parse and a
|
||||
// legacy rewrite produce. The rows layout additionally declares the per-row name column and
|
||||
// this statement has none to report: a division computes a value of its own and its inputs
|
||||
// are aggregates, so every row's name is null. That column is therefore the one difference
|
||||
// from the existing route's schema.
|
||||
assert_eq!(
|
||||
native_columns.len(),
|
||||
legacy_columns.len() + 1,
|
||||
@@ -11710,13 +11933,19 @@ async fn native_promql_rewrites_the_label_count_values_writes() {
|
||||
);
|
||||
}
|
||||
|
||||
/// `topk`/`bottomk` key their groups by series labels, while the existing plan also accepts a value
|
||||
/// field there and forms one group per sample value. A `by(<value field>)` therefore keeps every
|
||||
/// group's winner, and the native route has to report the same rows: the union below it reports its
|
||||
/// series, and the window above it partitions by the same value column. The rows are what this
|
||||
/// asserts - one winner per value group - rather than that no native node answers the union.
|
||||
/// `topk`/`bottomk` key their groups by series labels, and `by (val)` names no label a series
|
||||
/// carries: `val` is the value field, not a label. Prometheus therefore puts every series of the
|
||||
/// input into one empty group and answers only that group's winner, so the statement returns one
|
||||
/// row rather than one per sample value.
|
||||
///
|
||||
/// Both routes are about that grouping, but only the native one answers Prometheus here. The value
|
||||
/// column is named after the expression, so a `by (val)` over it selects nothing and the one empty
|
||||
/// group keeps the global winner. The existing plan still partitions its window by the `val` column
|
||||
/// it publishes, one group per sample value; that divergence is known and is not pinned here. What
|
||||
/// the native route must keep is the answer, with the union below it running as one native node
|
||||
/// rather than a legacy join.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() {
|
||||
async fn native_promql_ranks_a_value_field_grouping_as_one_group() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let (standalone, observer) = native_test_instance(GreptimeDbStandaloneBuilder::new(
|
||||
"native_promql_ranking_value_field",
|
||||
@@ -11748,28 +11977,25 @@ async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() {
|
||||
.await;
|
||||
|
||||
let pair = r#"native_rank_demo{host="x"} or native_rank_other{host="y"}"#;
|
||||
// `by (val)` forms two groups of one sample each, so both series survive: two rows, one winner
|
||||
// per value group, on both routes.
|
||||
for query in [
|
||||
format!("topk by (val) (1, {pair})"),
|
||||
format!("bottomk by (val) (1, {pair})"),
|
||||
// No series carries a `val` label, so the grouping is empty: every series lands in that one
|
||||
// group and the answer is its winner, `y` at 8 for `topk` and `x` at 2 for `bottomk`.
|
||||
for (query, expected) in [
|
||||
(
|
||||
format!("topk by (val) (1, {pair})"),
|
||||
vec![(0, "y".to_string(), 8.0)],
|
||||
),
|
||||
(
|
||||
format!("bottomk by (val) (1, {pair})"),
|
||||
vec![(0, "x".to_string(), 2.0)],
|
||||
),
|
||||
] {
|
||||
let sql = format!("TQL EVAL (0, 0, '1s') {query}");
|
||||
observer.clear();
|
||||
let legacy_rows = native_result_rows(native_tql_output(&instance, &sql, false).await).await;
|
||||
assert_eq!(
|
||||
legacy_rows,
|
||||
vec![(0, "x".to_string(), 2.0), (0, "y".to_string(), 8.0)],
|
||||
"{query}: the existing plan keeps one row per value group"
|
||||
);
|
||||
|
||||
observer.clear();
|
||||
let native_rows = native_result_rows(native_tql_output(&instance, &sql, true).await).await;
|
||||
let root = datafusion::physical_plan::displayable(observer.root_plan().as_ref())
|
||||
.indent(true)
|
||||
.to_string();
|
||||
// The union has to stay native rather than become the legacy join, and it must not fold the
|
||||
// two value groups into one.
|
||||
// The union has to stay native rather than become the legacy join.
|
||||
assert!(
|
||||
observer.fused().is_some(),
|
||||
"{query}: the union runs as one fused node: {root}"
|
||||
@@ -11778,7 +12004,7 @@ async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() {
|
||||
!root.contains("JoinExec"),
|
||||
"{query}: the union is not a legacy join: {root}"
|
||||
);
|
||||
assert_eq!(native_rows, legacy_rows, "{query}: full rows");
|
||||
assert_eq!(native_rows, expected, "{query}: full rows");
|
||||
}
|
||||
|
||||
// A label grouping is what the native node does honor, so it still runs natively and keeps every
|
||||
@@ -12090,7 +12316,16 @@ async fn native_promql_delta_temporality_rate_and_increase_match_legacy() {
|
||||
observer.clear();
|
||||
let (columns, rows) =
|
||||
native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
|
||||
assert_eq!(columns, vec!["__name__", "host", "ts", "val"]);
|
||||
// The value is named after the statement's expression, not `val`, and the rows answer
|
||||
// reports the per-row name column among its labels; a division drops the name.
|
||||
let mut expected_columns = vec![
|
||||
"__name__".to_string(),
|
||||
"host".to_string(),
|
||||
"ts".to_string(),
|
||||
expression_name(&query),
|
||||
];
|
||||
expected_columns.sort();
|
||||
assert_eq!(columns, expected_columns, "{query}");
|
||||
assert_eq!(rows.len(), 1, "{query}: {rows:?}");
|
||||
assert_eq!(rows[0].0, 120_000, "{query}");
|
||||
assert_eq!(
|
||||
@@ -12792,9 +13027,10 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() {
|
||||
// `count_values` names a label no scan carries, so the statement keeps a plan that reports it.
|
||||
let sql = "TQL EVAL (0, 0, '1s') count_values(\"bucket\", vector(1))";
|
||||
// The native route answers `count_values` itself now, so this is its own answer: one series at
|
||||
// t=0 whose generated label is "1" and whose count is 1. The aggregate drops the metric name -
|
||||
// as Prometheus does - so the declared name column is there and null, and it is the one column
|
||||
// the existing planner's answer does not have; the two headers differ while the numbers agree.
|
||||
// t=0 whose generated label is "1" and whose count is 1. The value column is named after the
|
||||
// expression, and the aggregate drops the metric name - as Prometheus does - so the declared
|
||||
// name column is there and null; it is the one column the existing planner's answer does not
|
||||
// have, and is left out of the columns compared below.
|
||||
let native = named_rows_text(
|
||||
native_tql_output(&instance, sql, true).await,
|
||||
ReportedName::Nothing,
|
||||
@@ -12802,11 +13038,11 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() {
|
||||
.await;
|
||||
assert_eq!(
|
||||
native,
|
||||
"+---------------------+-------+--------+\
|
||||
\n| time | value | bucket |\
|
||||
\n+---------------------+-------+--------+\
|
||||
\n| 1970-01-01T00:00:00 | 1.0 | 1 |\
|
||||
\n+---------------------+-------+--------+"
|
||||
"+---------------------+-----------------------------------+--------+\
|
||||
\n| time | count_values(\"bucket\", vector(1)) | bucket |\
|
||||
\n+---------------------+-----------------------------------+--------+\
|
||||
\n| 1970-01-01T00:00:00 | 1.0 | 1 |\
|
||||
\n+---------------------+-----------------------------------+--------+"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13618,10 +13854,16 @@ async fn native_promql_recursive_matching_segments_match_legacy() {
|
||||
expected, native,
|
||||
"matching on host drops job from output labels"
|
||||
);
|
||||
assert_eq!(
|
||||
vec!["__name__", "host", "ts", "val"],
|
||||
columns.iter().map(String::as_str).collect::<Vec<_>>()
|
||||
);
|
||||
// The value is named after the statement's expression, and the rows answer reports the
|
||||
// per-row name column among its labels.
|
||||
let mut expected_columns = vec![
|
||||
"__name__".to_string(),
|
||||
"host".to_string(),
|
||||
"ts".to_string(),
|
||||
expression_name(query),
|
||||
];
|
||||
expected_columns.sort();
|
||||
assert_eq!(expected_columns, columns);
|
||||
}
|
||||
|
||||
/// Segments are cut where the physical planner would otherwise have to gather a per-series prefix.
|
||||
@@ -14199,18 +14441,17 @@ async fn native_promql_real_query_q07_throttled_periods() {
|
||||
.iter()
|
||||
.all(|debug| debug.contains("FusedSeries"))
|
||||
);
|
||||
// Known computed-column naming gap from the ungrouped two-source case; values agree.
|
||||
// The two routes name the computed value column differently - the existing planner after its
|
||||
// own rewrite, the native route after the expression - and the labelled rows compared above
|
||||
// are what both report.
|
||||
assert!(
|
||||
legacy_columns
|
||||
.iter()
|
||||
.any(|column| column.contains("prom_increase"))
|
||||
);
|
||||
assert_eq!(
|
||||
native_columns
|
||||
.iter()
|
||||
.filter(|column| *column == "val")
|
||||
.count(),
|
||||
1
|
||||
assert!(
|
||||
native_columns.contains(&expression_name(query)),
|
||||
"Q07 native value column: {native_columns:?}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -14316,14 +14557,13 @@ async fn native_promql_real_query_q04_namespace_cpu_requests() {
|
||||
.iter()
|
||||
.all(|debug| debug.contains("FusedSeries"))
|
||||
);
|
||||
// Known computed-column naming gap from the ungrouped two-source case; values agree.
|
||||
// The two routes name the computed value column differently - the existing planner after its
|
||||
// own rewrite, the native route after the expression - and the labelled rows compared above
|
||||
// are what both report.
|
||||
assert!(legacy_columns.iter().any(|column| column.contains("max(")));
|
||||
assert_eq!(
|
||||
native_columns
|
||||
.iter()
|
||||
.filter(|column| *column == "val")
|
||||
.count(),
|
||||
1
|
||||
assert!(
|
||||
native_columns.contains(&expression_name(query)),
|
||||
"Q04 native value column: {native_columns:?}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -14668,11 +14908,20 @@ async fn native_promql_dynamic_topk_matches_legacy() {
|
||||
"a bare topk above no aggregate runs natively: {bare_sql}"
|
||||
);
|
||||
// The same ranking answers the same rows on both routes; the rows keep the name of the series
|
||||
// `topk` selected, which this route states in its own declared column.
|
||||
let mut named_columns = bare_legacy.0.clone();
|
||||
named_columns.push("__name__".to_string());
|
||||
named_columns.sort();
|
||||
assert_eq!(bare_native.0, named_columns, "{bare_sql}: columns");
|
||||
// `topk` selected, which this route states in its own declared column. The two routes name the
|
||||
// one value column differently - the existing planner keeps the tree's `val`, the native route
|
||||
// names it after the expression - so the columns are the existing answer's, with that name and
|
||||
// the name column.
|
||||
let mut expected_columns = bare_legacy
|
||||
.0
|
||||
.iter()
|
||||
.filter(|column| *column != "val")
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
expected_columns.push("__name__".to_string());
|
||||
expected_columns.push(expression_name("topk(3, native_dyn_topk)"));
|
||||
expected_columns.sort();
|
||||
assert_eq!(bare_native.0, expected_columns, "{bare_sql}: columns");
|
||||
assert_eq!(
|
||||
bare_native.1,
|
||||
named_rows(bare_legacy.1, "native_dyn_topk"),
|
||||
|
||||
@@ -1881,19 +1881,17 @@ fn result_columns(schema: &SchemaRef) -> Vec<(String, DataType, bool)> {
|
||||
///
|
||||
/// The tags are stated by the caller rather than read off the result: a vector that matches no
|
||||
/// series reports no row to read them from, and still answers with its tag columns.
|
||||
fn assert_native_result_schema(schema: &SchemaRef, tags: &[&str], context: &str) {
|
||||
// The time index and the one field the tree reports, then one `Utf8` column per tag.
|
||||
fn assert_native_result_schema(schema: &SchemaRef, tags: &[&str], value_name: &str, context: &str) {
|
||||
// The time index and the one field the tree reports, then one `Utf8` column per tag. The
|
||||
// native route names the value column after the expression, so the caller states that name;
|
||||
// `value`/`val` are the legacy route's names.
|
||||
let mut columns = vec![
|
||||
(
|
||||
if tags.is_empty() { "time" } else { "ts" }.to_string(),
|
||||
DataType::Timestamp(TimeUnit::Millisecond, None),
|
||||
false,
|
||||
),
|
||||
(
|
||||
if tags.is_empty() { "value" } else { "val" }.to_string(),
|
||||
DataType::Float64,
|
||||
true,
|
||||
),
|
||||
(value_name.to_string(), DataType::Float64, true),
|
||||
];
|
||||
for name in tags {
|
||||
columns.push(((*name).to_string(), DataType::Utf8, true));
|
||||
@@ -2218,9 +2216,13 @@ async fn assert_raw_scan_fused_formats(observer: &NativeFusedObserver, frontend:
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.unwrap();
|
||||
// The native route names its value column after the expression, so the
|
||||
// type is what identifies it; `val` is only the legacy route's name.
|
||||
let values = batch
|
||||
.column_by_name("val")
|
||||
.unwrap()
|
||||
.columns()
|
||||
.iter()
|
||||
.find(|column| column.data_type() == &DataType::Float64)
|
||||
.expect("one Float64 value column")
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.unwrap();
|
||||
@@ -2744,7 +2746,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
|
||||
};
|
||||
let (plan, schema, rows) = promql_output(frontend, query, true, start, end).await;
|
||||
assert_rows_eq(&rows, &expected, query);
|
||||
assert_native_result_schema(&schema, tags, query);
|
||||
assert_native_result_schema(&schema, tags, query, query);
|
||||
let fused = assert_native_plan(&plan, scan_partitions, query);
|
||||
assert_eq!(
|
||||
fused["metrics"]["native.root.output_values"].as_u64(),
|
||||
@@ -2769,13 +2771,33 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
|
||||
);
|
||||
assert_rows_eq(&legacy_rows, &expected, query);
|
||||
assert_legacy_result_schema(&legacy_schema, tags, query);
|
||||
// Both constant cases preserve the legacy time/value schema; computed vector columns are
|
||||
// compared by role.
|
||||
// Both routes report the time column and one Float64 value. The native route names that
|
||||
// value after the expression, so the constant cases agree by role, not by header name.
|
||||
if scan_partitions.is_none() {
|
||||
assert_eq!(
|
||||
result_columns(&legacy_schema),
|
||||
result_columns(&schema),
|
||||
"`{query}`: the two routes must report the same constant schema"
|
||||
2,
|
||||
schema.fields().len(),
|
||||
"`{query}`: native constant schema"
|
||||
);
|
||||
assert_eq!(
|
||||
2,
|
||||
legacy_schema.fields().len(),
|
||||
"`{query}`: legacy constant schema"
|
||||
);
|
||||
assert_eq!(
|
||||
schema.field(0).data_type(),
|
||||
legacy_schema.field(0).data_type(),
|
||||
"`{query}`: time column type"
|
||||
);
|
||||
assert_eq!(
|
||||
&DataType::Float64,
|
||||
schema.field(1).data_type(),
|
||||
"`{query}`: native value type"
|
||||
);
|
||||
assert_eq!(
|
||||
&DataType::Float64,
|
||||
legacy_schema.field(1).data_type(),
|
||||
"`{query}`: legacy value type"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2788,7 +2810,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
|
||||
observer.clear();
|
||||
let (plan, schema, rows) = promql_output(frontend, query, true, "0", "2").await;
|
||||
assert!(rows.is_empty(), "`{query}` reported {} rows", rows.len());
|
||||
assert_native_result_schema(&schema, &["host"], query);
|
||||
assert_native_result_schema(&schema, &["host"], query, query);
|
||||
assert!(
|
||||
executes_fused_series(&plan),
|
||||
"`{query}` did not execute the fused route"
|
||||
|
||||
Reference in New Issue
Block a user