feat(promql): name native result columns from expressions (#10231)

Use the statement expression for native value column names, resolve legacy value columns by type rather than by name, and reject label-function destination collisions instead of dropping a value. Keep matrix roots named after the whole statement, preserve an Int64 __tsid value, and propagate schema-collision errors from canonical boundaries.

Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
2026-11-17 12:53:55 +08:00
committed by GitHub
parent 3bdde3d339
commit 48bb6beadd
11 changed files with 1456 additions and 464 deletions
+85 -14
View File
@@ -377,8 +377,8 @@ struct FusedSeriesPayload {
impl FusedSeries {
/// Serializes the native tree and the binding required to reconstruct this plan node.
/// The PromQL text is included for diagnostics only and never drives execution; children
/// remain ordinary Substrait inputs.
/// The PromQL text identifies expression-named output fields; children remain ordinary
/// Substrait inputs.
///
/// This is only the node's own encoding. Whether a planner is allowed to place this node on
/// another machine is a separate decision, made by the planner and the transport, not here.
@@ -497,6 +497,7 @@ impl FusedSeries {
&node.tree,
&node.output_tags()?,
&node.binding.time_index,
&node.promql,
)?;
}
Ok(node)
@@ -1051,6 +1052,22 @@ impl FusedSeries {
/// from the whole tree - a match narrows them, `or` joins both sides' and a wrapper around
/// either keeps them - and the output schema is where it records that.
fn output_tags(&self) -> DataFusionResult<Vec<String>> {
if matches!(self.tree.root().output, NativeType::Scalar(_)) {
return Ok(Vec::new());
}
if self.format == FusedOutputFormat::PlainRows {
// Roles are types here too: a value is a float or a native histogram, so every other
// non-time column of a row table is one of this output's labels, whatever its name.
return Ok(self
.output_schema
.fields()
.iter()
.filter(|field| {
field.name() != &self.binding.time_index && field.data_type() == &DataType::Utf8
})
.map(|field| field.name().clone())
.collect());
}
let field_names = self
.tree
.root()
@@ -1732,6 +1749,7 @@ fn validate_plain_output_schema(
tree: &NativeTree,
tags: &[String],
time: &str,
promql: &str,
) -> DataFusionResult<()> {
let fields = match &tree.root().output {
NativeType::Samples(s) => s.fields.to_vec(),
@@ -1755,7 +1773,7 @@ fn validate_plain_output_schema(
));
}
};
classify_output(schema, &fields, tags, time)?;
classify_output(schema, &fields, tags, time, promql)?;
Ok(())
}
@@ -1844,13 +1862,73 @@ fn scalar_root_fields(
}])
}
/// Whether a column of this type carries a field's samples.
///
/// A role is a type: a field holds floats or the canonical native histogram struct, a tag is
/// `Utf8` and time is a timestamp. A name says nothing about which of those a column is.
fn is_value_column(data_type: &DataType) -> bool {
data_type == &DataType::Float64 || data_type == &native_histogram_value_type().as_arrow_type()
}
/// Whether `names` name exactly the columns in `value_names`, one name per column.
fn names_value_columns(value_names: &[&str], names: &[String]) -> bool {
if names.len() != value_names.len() {
return false;
}
let mut declared = names.iter().map(String::as_str).collect::<Vec<_>>();
declared.sort_unstable();
let mut carried = value_names.to_vec();
carried.sort_unstable();
declared == carried
}
/// Names the fields in a row schema, in `series_fields` order.
///
/// A caller names a value column either after the expression that produced it or after the tree's
/// field, so the typed value columns are what decide which: the expression names are used only
/// when they name exactly those columns, and the field names when those do. Deciding by type keeps
/// a tag that shares a field's name from being read as that field. The result stays in
/// `series_fields` order because a name is bound back to its field by that index, so a reordered
/// output still reads every column from its own field.
fn output_field_names(
output_schema: &SchemaRef,
series_fields: &[FieldSpec],
promql: &str,
) -> Vec<String> {
let value_names = output_schema
.fields()
.iter()
.filter(|field| is_value_column(field.data_type()))
.map(|field| field.name().as_str())
.collect::<Vec<_>>();
let expression_names = series_fields
.iter()
.map(|field| {
if series_fields.len() == 1 {
promql.to_string()
} else {
format!("{promql}.{}", field.name)
}
})
.collect::<Vec<_>>();
if names_value_columns(&value_names, &expression_names) {
return expression_names;
}
series_fields
.iter()
.map(|field| field.name.to_string())
.collect()
}
/// Classifies every output column, rejecting anything the conversion cannot produce.
pub(crate) fn classify_output(
output_schema: &SchemaRef,
series_fields: &[FieldSpec],
tags: &[String],
time_column: &str,
promql: &str,
) -> DataFusionResult<Vec<OutputColumn>> {
let field_names = output_field_names(output_schema, series_fields, promql);
if output_schema
.fields()
.iter()
@@ -1873,11 +1951,6 @@ pub(crate) fn classify_output(
"tag column {tag:?} of FusedSeriesExec cannot be the time column"
)));
}
if series_fields.iter().any(|field| field.name.as_ref() == tag) {
return Err(DataFusionError::Plan(format!(
"tag column {tag:?} of FusedSeriesExec clashes with a series field"
)));
}
}
let mut time_seen = false;
@@ -1909,10 +1982,7 @@ pub(crate) fn classify_output(
classification.push(OutputColumn::Time);
continue;
}
if let Some(index) = series_fields
.iter()
.position(|field| field.name.as_ref() == name)
{
if let Some(index) = field_names.iter().position(|field| field == name) {
if std::mem::replace(&mut field_seen[index], true) {
return Err(DataFusionError::Plan(format!(
"duplicate field column {name:?} in FusedSeriesExec output schema"
@@ -1977,7 +2047,7 @@ pub(crate) fn classify_output(
if !seen {
return Err(DataFusionError::Plan(format!(
"missing field column {:?} in FusedSeriesExec output schema",
series_fields[index].name
field_names[index]
)));
}
}
@@ -2485,7 +2555,7 @@ impl FusedSeriesExec {
Vec::new()
} else if scalar_root {
let scalar_fields = scalar_root_fields(&output_schema, &tags, &time_column)?;
classify_output(&output_schema, &scalar_fields, &tags, &time_column)?
classify_output(&output_schema, &scalar_fields, &tags, &time_column, &promql)?
} else {
classify_output(
&output_schema,
@@ -2496,6 +2566,7 @@ impl FusedSeriesExec {
.fields,
&tags,
&time_column,
&promql,
)?
};
let partition_count = inputs.first().map_or(1, |(input, _, _)| {
@@ -2599,6 +2599,149 @@ fn assert_decode_error(bytes: &[u8], expected: &str) {
);
}
#[test]
fn renamed_reordered_value_columns_keep_field_identity_after_decode() {
let tree = rate_tree_of(two_field_schema());
let schema = Arc::new(Schema::new(vec![
Field::new(format!("{PROMQL}.{OTHER}"), DataType::Float64, true),
Field::new(HOST, DataType::Utf8, true),
Field::new(
TIME,
DataType::Timestamp(TimeUnit::Millisecond, None),
false,
),
Field::new(format!("{PROMQL}.{VALUE}"), DataType::Float64, true),
]));
let node = FusedSeries::try_new(
Arc::from(PROMQL),
tree.clone(),
binding_of(&tree, two_field_schema(), vec![HOST.to_string()]),
vec![input_plan(frame_schema_of(&two_field_schema()))],
Arc::new(datafusion::common::DFSchema::try_from(schema.as_ref().clone()).unwrap()),
)
.unwrap();
let decoded = FusedSeries::deserialize(&node.serialize().unwrap()).unwrap();
assert_eq!(node.schema(), decoded.schema());
let tags = decoded.output_tags().unwrap();
assert_eq!(vec![HOST], tags);
let classification =
super::classify_output(&schema, &two_field_schema().fields, &tags, TIME, PROMQL).unwrap();
let chunk = EvaluatedChunk::try_new(
Series {
metric: SeriesLabels::try_new(None, Vec::new()).unwrap(),
time: TimeAxis::from_grid_spans(grid(), vec![GridSpan::new(0, 1)]).unwrap(),
fields: SeriesFields::try_from_vec(vec![
Samples::from_float(Float64Array::from(vec![10.0])),
Samples::from_float(Float64Array::from(vec![20.0])),
])
.unwrap(),
},
true,
)
.unwrap();
let batch = series_to_record_batch(&classification, &tags, &schema, &chunk).unwrap();
for (name, expected) in [(OTHER, 20.0), (VALUE, 10.0)] {
let values = batch
.column_by_name(&format!("{PROMQL}.{name}"))
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
assert_eq!(expected, values.value(0));
}
}
/// A tag that carries the expression's name is a tag, not the statement's value.
///
/// This is an old-style row schema: the statement is `host`, the schema names its one value
/// `field_0` and also carries the `host` tag. Deciding the value by whether a name occurs
/// anywhere in the schema would pick the tag as the value; the typed columns put the value at
/// `field_0` and the tag at `host`.
#[test]
fn a_tag_named_after_the_expression_is_not_the_value() {
let schema = Arc::new(Schema::new(vec![
Field::new(
TIME,
DataType::Timestamp(TimeUnit::Millisecond, None),
false,
),
Field::new("field_0", DataType::Float64, true),
Field::new(HOST, DataType::Utf8, true),
]));
let fields = [FieldSpec {
name: "field_0".into(),
}];
let classification =
super::classify_output(&schema, &fields, &[HOST.to_string()], TIME, HOST).unwrap();
assert!(
matches!(
classification[1],
OutputColumn::Field {
index: 0,
kind: FieldKind::Float
}
),
"{classification:?}"
);
assert!(
matches!(classification[2], OutputColumn::Tag(0)),
"{classification:?}"
);
}
/// A reordered output still binds every value column to its own field.
///
/// The expression names are matched as a set against the typed value columns, and the result is
/// kept in `series_fields` order - here `other` before `value` - because that order is what the
/// classification indexes the chunk's fields by. Names taken in output order would bind the first
/// column to `value`.
#[test]
fn reordered_expression_names_bind_fields_in_series_order() {
let schema = Arc::new(Schema::new(vec![
Field::new(
TIME,
DataType::Timestamp(TimeUnit::Millisecond, None),
false,
),
Field::new(format!("{PROMQL}.{OTHER}"), DataType::Float64, true),
Field::new(HOST, DataType::Utf8, true),
Field::new(format!("{PROMQL}.{VALUE}"), DataType::Float64, true),
]));
let series_schema = two_field_schema();
let classification = super::classify_output(
&schema,
&series_schema.fields,
&[HOST.to_string()],
TIME,
PROMQL,
)
.unwrap();
assert!(
matches!(
classification[1],
OutputColumn::Field {
index: 1,
kind: FieldKind::Float
}
),
"{classification:?}"
);
assert!(
matches!(classification[2], OutputColumn::Tag(0)),
"{classification:?}"
);
assert!(
matches!(
classification[3],
OutputColumn::Field {
index: 0,
kind: FieldKind::Float
}
),
"{classification:?}"
);
}
#[test]
fn plain_rows_decode_rejects_invalid_schemas() {
let node = fused(rate_tree());
@@ -2649,16 +2792,62 @@ fn plain_rows_decode_rejects_invalid_schemas() {
false,
),
Field::new(VALUE, DataType::Float64, true),
// A column's role is its type: a float here is not a tag, so this payload names a
// column the conversion cannot produce.
Field::new(HOST, DataType::Float64, true),
Field::new(NAME, DataType::Utf8, true),
]),
"must be Utf8",
"unknown column",
),
];
for (schema, expected) in malformed_schemas {
assert_decode_error(&mutate_schema(&bytes, schema), expected);
}
let renamed = mutate_envelope_payload(&bytes, |payload| {
let mut fields = payload.output_schema.fields().to_vec();
let index = fields
.iter()
.position(|field| field.name() == VALUE)
.unwrap();
fields[index] = Arc::new(Field::new(PROMQL, DataType::Float64, true));
payload.output_schema = Schema::new(fields);
});
assert!(FusedSeries::deserialize(&renamed).is_ok());
// A value column is identified by type, so this payload types its one float column as neither
// a float nor a native histogram and names no value column the tree can read; a float value
// column that cannot be null is refused for its nullability, because a grid point may be
// missing.
for (data_type, nullable, expected) in [
(DataType::Int64, true, "unknown column"),
(DataType::Float64, false, "must be nullable"),
] {
let malformed = mutate_envelope_payload(&renamed, |payload| {
let mut fields = payload.output_schema.fields().to_vec();
let index = fields
.iter()
.position(|field| field.name() == PROMQL)
.unwrap();
fields[index] = Arc::new(Field::new(PROMQL, data_type, nullable));
payload.output_schema = Schema::new(fields);
});
assert_decode_error(&malformed, expected);
}
// A scalar's value column is its one non-time column, so the wrong type is reported as a kind
// the conversion cannot produce.
let scalar_wrong_kind = mutate_envelope_payload(&scalar_bytes, |payload| {
let mut fields = payload.output_schema.fields().to_vec();
let index = fields
.iter()
.position(|field| field.name() != TIME)
.expect("a scalar carries one value column");
let name = fields[index].name().clone();
fields[index] = Arc::new(Field::new(name, DataType::Int64, true));
payload.output_schema = Schema::new(fields);
});
assert_decode_error(&scalar_wrong_kind, "must be Float64");
let scalar_decoded = FusedSeries::deserialize(&scalar_bytes).unwrap();
assert_eq!(scalar_decoded.schema(), scalar.schema());
+34 -10
View File
@@ -21,8 +21,8 @@
//! nodes, recursing into a cut that wraps a whole-vector consumer.
//!
//! Everything here reads the bound tree alone. The columns a node publishes are the planner's
//! answer, so the builders ask for them through [`OutputTags`] and [`RootSchema`] instead of
//! deriving them a second time.
//! answer, so the builders ask for them through [`OutputTags`], [`FieldNames`] and [`RootSchema`]
//! instead of deriving them a second time.
use std::sync::Arc;
@@ -56,11 +56,20 @@ use crate::vector_match::Cardinality;
/// rather than deriving the columns a second time.
pub type OutputTags = fn(&NativeNode, &[Vec<String>]) -> Option<Vec<String>>;
/// The row schema a statement root publishes, or `None` when its columns cannot be named once.
/// Derives the value column names a statement's expression gives the compiled `fields`.
///
/// Like [`OutputTags`], this is the planner's answer: a segment publishes the values its tree
/// computes under the names the statement gave them, so the builders ask through this leaf instead
/// of publishing the compiler's own field names. The tree alone does not carry the statement's
/// expression, which is why the expression text is an input rather than something derived here.
pub type FieldNames = fn(&str, &[String]) -> Vec<String>;
/// The row schema a statement root publishes for `promql`, or `None` when its columns cannot be
/// named once.
///
/// Like [`OutputTags`], this is the caller's answer: a segment that publishes rows publishes
/// exactly the columns the planner would have published for the same tree.
pub type RootSchema = fn(&NativeTree, &[Vec<String>], &str, FieldKind) -> Option<DFSchemaRef>;
/// exactly the columns the planner would have published for the same statement.
pub type RootSchema = fn(&str, &NativeTree, &[Vec<String>], &str, FieldKind) -> Option<DFSchemaRef>;
/// Whether `node` reads samples point-wise, so that one partitioned execution is one answer.
///
@@ -496,8 +505,8 @@ pub const NATIVE_ORDERING: SeriesOrdering = SeriesOrdering::Contiguous;
/// The columns a [`FusedSeries`] node produces.
///
/// With `experimental_native_promql` enabled, a value column is named after its compiled root
/// field (for example, `field_0`), not legacy planner expression text. Legacy headers in
/// With `experimental_native_promql` enabled, a value column is named by the caller's [`FieldNames`]
/// answer for the statement's expression, not legacy planner expression text. Legacy headers in
/// `tests/cases/**` are unaffected because the native route is opt-in and off by default.
///
/// The shape is the one the existing unary pipeline consumes — the time index, then the series
@@ -623,6 +632,7 @@ pub fn canonical_boundary_stages(
format: FusedOutputFormat,
root_kind: FieldKind,
output_tags_of: OutputTags,
field_names: FieldNames,
) -> Option<(LogicalPlan, Vec<String>)> {
let NativeExpr::Merge { input } = &tree.root().expr else {
return None;
@@ -644,7 +654,12 @@ pub fn canonical_boundary_stages(
// declares it exactly as the other rows layouts do. The tags stay the walk's own: they name the
// labels the frames carry, and the binding below is built from them.
let row_tags = with_row_name_column(tree.root(), output_tags.clone());
let output_schema = fused_output_schema(&fields, &row_tags, time_index, root_kind).ok()?;
// The values are named after the expression, as in every other rows layout of this statement.
// A schema this boundary cannot declare declines here: the caller's error type is not one this
// crate can build, so a name collision is reported by declining rather than as an internal
// error, and the caller keeps the plan it would have chosen.
let value_names = field_names(&promql, &fields);
let output_schema = fused_output_schema(&value_names, &row_tags, time_index, root_kind).ok()?;
// The producer: the statement's own tree, run per partition, exporting a frame. It runs above
// the regions, where its router collects each label set from every region's contribution.
@@ -766,6 +781,7 @@ fn build_evaluated_segment(
time_index: &str,
root_kind: FieldKind,
output_tags_of: OutputTags,
field_names: FieldNames,
) -> Option<(LogicalPlan, Vec<String>)> {
let mut sources = Vec::new();
collect_segment_inputs(node, &mut sources);
@@ -829,6 +845,7 @@ fn build_evaluated_segment(
FusedOutputFormat::SeriesFrame,
root_kind,
output_tags_of,
field_names,
)?;
return Some((plan, tags));
}
@@ -959,7 +976,8 @@ fn build_inner_segment(
/// per-series prefixes do not need that gather, so each becomes a segment of its own and keeps the
/// partitions it was scanned with. Global consumers and canonical merges are cut in their original
/// order, and recursive segments publish frames for their parents to consume. Only the statement
/// root publishes rows.
/// root publishes rows, and it does so under the statement's own value names: [`FieldNames`]
/// answers those, and the rows schema asks [`RootSchema`] for the rest of the layout.
#[allow(clippy::too_many_arguments, clippy::type_complexity)]
pub fn segment_tree(
promql: Arc<str>,
@@ -977,6 +995,7 @@ pub fn segment_tree(
root_kind: FieldKind,
remote_raw_boundary: bool,
output_tags_of: OutputTags,
field_names: FieldNames,
root_schema: RootSchema,
) -> Option<LogicalPlan> {
// Only a tree the analyzer gathers is worth splitting: its per-series prefixes then run where
@@ -1061,6 +1080,7 @@ pub fn segment_tree(
root_kind,
remote_raw_boundary,
output_tags_of,
field_names,
root_schema,
);
match plan {
@@ -1096,6 +1116,7 @@ pub fn segment_tree(
time_index,
root_kind,
output_tags_of,
field_names,
)
});
let (plan, cut_tags) = built?;
@@ -1164,6 +1185,7 @@ pub fn segment_tree(
format,
root_kind,
output_tags_of,
field_names,
)
.map(|(plan, _)| plan);
}
@@ -1176,7 +1198,9 @@ pub fn segment_tree(
return None;
}
let output_schema = match format {
FusedOutputFormat::PlainRows => root_schema(&parent, &child_tags, time_index, root_kind)?,
FusedOutputFormat::PlainRows => {
root_schema(&promql, &parent, &child_tags, time_index, root_kind)?
}
FusedOutputFormat::SeriesFrame => {
let Ok(frame) = FusedSeries::series_frame_schema(&parent) else {
return None;
+19 -21
View File
@@ -1866,9 +1866,8 @@ mod tests {
);
}
/// A statement that reads no table plans natively, and there is no region to place anything in:
/// the frontend evaluates the fused node under the native column names rather than the ones the
/// existing planner derives from an expression.
/// A statement that reads no table plans natively over the wire, where there is no region to
/// place anything in: the frontend evaluates the fused node with expression-named values.
#[tokio::test]
async fn test_remote_native_promql_plans_a_data_free_statement_natively() {
let engine = create_promql_test_engine_with(
@@ -1879,35 +1878,26 @@ mod tests {
false,
);
let query_ctx = native_discovery_query_ctx();
for (query, expected_columns, expected_values, expected_labels) in [
// `time()` answers the evaluation grid in seconds.
(
"time()",
&["time", "value"][..],
&[0.0, 5.0, 10.0][..],
&[][..],
),
for (query, expected_values, expected_labels, name_column) in [
// `time()` answers the evaluation grid in seconds. It is a scalar root: it publishes
// one value and no series, so its rows layout declares no name column.
("time()", &[0.0, 5.0, 10.0][..], &[][..], false),
// `vector(1)` is a samples root: its rows layout declares the per-row name column,
// which names no row because the vector carries no series identity.
(
"vector(1)",
&["time", "value", "__name__"][..],
&[1.0, 1.0, 1.0][..],
&[][..],
),
("vector(1)", &[1.0, 1.0, 1.0][..], &[][..], true),
(
r#"label_replace(vector(1), "host", "host1", "", "")"#,
&["time", "value", "host", "__name__"][..],
&[1.0, 1.0, 1.0][..],
&[("host", "host1")][..],
true,
),
// A synthesized `__tsid` is published under the statement's own columns: the stripping
// projection above the node is read through rather than mistaken for a remote plan.
(
r#"label_replace(vector(1), "__tsid", "x", "", "")"#,
&["time", "value", "__name__"][..],
&[1.0, 1.0, 1.0][..],
&[][..],
true,
),
] {
let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
@@ -1917,7 +1907,15 @@ mod tests {
!display.contains("greptime_value"),
"{query} keeps the native value column: {display}"
);
// A data-free statement has no input at all, so nothing is scanned or left empty.
// A data-free statement has no input at all, so nothing is scanned or left empty. The
// value is named after the expression, and a samples root's rows keep the name column
// last.
let expression = promql_parser::parser::parse(query).unwrap().to_string();
let mut expected_columns = vec!["time", expression.as_str()];
expected_columns.extend(expected_labels.iter().map(|(name, _)| *name));
if name_column {
expected_columns.push("__name__");
}
assert_eq!(
expected_columns,
plan.schema()
@@ -1931,7 +1929,7 @@ mod tests {
assert_regionless_physical_plan(&physical, query);
assert_eq!(
expected_values,
numeric_values(&batches, "value").as_slice(),
numeric_values(&batches, &expression).as_slice(),
"{query}: {batches:?}"
);
for (label, value) in expected_labels {
+11 -1
View File
@@ -58,6 +58,15 @@ pub enum Error {
location: Location,
},
#[snafu(display(
"duplicate native PromQL output column name(s): {names}; time index, value and tag columns must have distinct names"
))]
DuplicateOutputColumns {
names: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Unexpected plan or expression: {}", desc))]
UnexpectedPlanExpr {
desc: String,
@@ -281,7 +290,8 @@ impl ErrorExt for Error {
| SystemTimeOutOfRange { .. }
| AtModifierTimestampOutOfRange { .. }
| InvalidRegularExpression { .. }
| InvalidDestinationLabelName { .. } => StatusCode::InvalidArguments,
| InvalidDestinationLabelName { .. }
| DuplicateOutputColumns { .. } => StatusCode::InvalidArguments,
UnknownTable { .. } => StatusCode::Internal,
+58 -20
View File
@@ -62,7 +62,7 @@ use datafusion_expr::expr_fn::when;
use datafusion_expr::utils::{conjunction, disjunction};
use datafusion_expr::{ExprSchemable, Literal, SortExpr, TableSource, ident, lit};
use datafusion_functions::core::coalesce;
use datatypes::arrow::datatypes::{DataType as ArrowDataType, TimeUnit as ArrowTimeUnit};
use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit as ArrowTimeUnit};
use datatypes::data_type::{ConcreteDataType, DataType as GreptimeDataType};
use itertools::Itertools;
use once_cell::sync::Lazy;
@@ -1435,13 +1435,11 @@ impl PromPlanner {
let right_ctx = self.ctx.clone();
(left, left_ctx, right, right_ctx)
};
let left_field_columns = left_context.field_columns.clone();
let left_time_index_column = left_context.time_index_column.clone();
self.ctx = left_context.clone();
let mut left_table_ref = self
.table_ref()
.unwrap_or_else(|_| TableReference::bare(""));
let right_field_columns = right_context.field_columns.clone();
let right_time_index_column = right_context.time_index_column.clone();
self.ctx = right_context.clone();
let mut right_table_ref = self
@@ -1450,7 +1448,8 @@ impl PromPlanner {
// Both operands are planned first because only the planned contexts tell a tag
// from a value field. The rewrite merely adds matchers to a selector, so the
// table reference, time index and field columns captured above stay valid.
// table reference and time index captured above stay valid. The field columns are
// read after this rewrite, because an operand it re-plans brings its own.
if let Some(rewritten) = matching_filters::propagate(
binary_expr,
&left_context.tag_columns,
@@ -1481,6 +1480,8 @@ impl PromPlanner {
self.ctx = right_context.clone();
}
let left_field_columns = left_context.field_columns.clone();
let right_field_columns = right_context.field_columns.clone();
let left_is_empty_metric = Self::is_empty_metric(&left_input);
let right_is_empty_metric = Self::is_empty_metric(&right_input);
@@ -4019,12 +4020,28 @@ impl PromPlanner {
query_engine_state,
)?;
// Reserve the current field columns except the `dst_label`.
for value in &self.ctx.field_columns {
if *value != dst_label {
let expr = DfExpr::Column(Column::from_name(value));
exprs.push(expr);
// A destination label is a tag, and a tag is published under its own name. A
// value field of that name would have to be dropped or renamed to make room, and
// neither is what the statement says, so the collision is the statement's error
// rather than a value quietly going missing.
if self
.ctx
.field_columns
.iter()
.any(|field| field == &dst_label)
{
return UnsupportedExprSnafu {
name: format!(
"label_join destination label {dst_label:?} collides with a value field"
),
}
.fail();
}
// Reserve the current field columns.
for value in &self.ctx.field_columns {
let expr = DfExpr::Column(Column::from_name(value));
exprs.push(expr);
}
// Remove it from tag columns if exists to avoid duplicated column names
@@ -4040,12 +4057,10 @@ impl PromPlanner {
if let Some((replace_expr, dst_label)) =
self.build_regexp_replace_label_expr(&mut other_input_exprs, input_schema)?
{
// Reserve the current field columns except the `dst_label`.
// Reserve the current field columns.
for value in &self.ctx.field_columns {
if *value != dst_label {
let expr = DfExpr::Column(Column::from_name(value));
exprs.push(expr);
}
let expr = DfExpr::Column(Column::from_name(value));
exprs.push(expr);
}
// Remove it from tag columns if exists to avoid duplicated column names
@@ -4375,6 +4390,22 @@ impl PromPlanner {
// Validate the destination label name
Self::validate_label_name(&dst_label)?;
// A destination label is a tag, and a tag is published under its own name. A value field
// of that name would have to be dropped or renamed to make room, so report the collision
// before any no-op path below can silently decline the rewrite.
if self
.ctx
.field_columns
.iter()
.any(|field| field == &dst_label)
{
return UnsupportedExprSnafu {
name: format!(
"label_replace destination label {dst_label:?} collides with a value field"
),
}
.fail();
}
let replacement = match other_input_exprs.pop_front() {
Some(DfExpr::Literal(ScalarValue::Utf8(Some(r)), _)) => r,
other => UnexpectedPlanExprSnafu {
@@ -6668,11 +6699,18 @@ impl PromPlanner {
fn strip_tsid_column(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
let schema = plan.schema();
if !schema
.fields()
.iter()
.any(|field| field.name() == DATA_SCHEMA_TSID_COLUMN_NAME)
{
// A series identifier is the internal `UInt64` key or the `Utf8` label synthesized from
// it, and only those two representations are stripped. The same name on any other column
// is the statement's own answer - `count(m)` aliased to `__tsid` answers an `Int64`, and a
// value is `Float64` or the canonical native histogram struct - so a name is not a role.
fn is_internal_tsid(field: &Field) -> bool {
field.name() == DATA_SCHEMA_TSID_COLUMN_NAME
&& matches!(
field.data_type(),
ArrowDataType::UInt64 | ArrowDataType::Utf8
)
}
if !schema.fields().iter().any(|field| is_internal_tsid(field)) {
return Ok(plan);
}
@@ -6680,7 +6718,7 @@ impl PromPlanner {
// the columns by their original qualified names.
let project_exprs = schema
.iter()
.filter(|(_, field)| field.name() != DATA_SCHEMA_TSID_COLUMN_NAME)
.filter(|(_, field)| !is_internal_tsid(field))
.map(|(qualifier, field)| {
DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone()))
})
+181 -73
View File
@@ -77,11 +77,11 @@ use promql_parser::parser::{
EvalStmt, Expr as PromExpr, MatrixSelector, ParenExpr, SubqueryExpr, VectorSelector,
};
use session::context::{MetricDiscoveryScope, QueryContextRef};
use snafu::ResultExt;
use snafu::{ResultExt, ensure};
use store_api::metric_engine_consts::{DATA_SCHEMA_TABLE_ID_COLUMN_NAME, METRIC_ENGINE_NAME};
use store_api::region_engine::RawLabelSource;
use crate::promql::error::{DataFusionPlanningSnafu, Result};
use crate::promql::error::{DataFusionPlanningSnafu, DuplicateOutputColumnsSnafu, Result};
use crate::promql::native_compile::{
InfoInput, bind_evaluated_binary, compile_selectors_with_grid, compile_statement_with_grid,
info_input, statement_grid, subquery_child_grid,
@@ -528,6 +528,8 @@ impl PromPlanner {
{
return Ok(None);
}
// The expression names the value columns, and it is the label the node reports.
let expression = expr.to_string();
// What a consumer of this composition reads, and the layout the node publishes. The row
// layout names one column per value; the frame layout is the evaluated frame this tree's
// own root declares, which is what a frame consumer decodes.
@@ -542,8 +544,13 @@ impl PromPlanner {
};
let output_schema = match format {
FusedOutputFormat::PlainRows => {
let Some(schema) =
Self::fused_root_schema(&tree, &source_tags, &left_time, root_kind)
let Some(schema) = Self::fused_root_schema(
&expression,
&tree,
&source_tags,
&left_time,
root_kind,
)?
else {
return Ok(None);
};
@@ -562,7 +569,7 @@ impl PromPlanner {
}
};
let Ok(node) = FusedSeries::try_new_with_format(
Arc::from(expr.to_string()),
Arc::from(expression.as_str()),
tree,
FusedSeriesBinding {
grid: Arc::clone(&left_grid),
@@ -586,8 +593,8 @@ impl PromPlanner {
/// Plans `prom_expr` as one native node, or returns `None` to keep the existing plan.
///
/// The compiler's bound tree and selector schemas determine the native output names; the existing
/// planner is not consulted for capability or schema approval.
/// The expression names the value columns; the compiler's bound tree and selector schemas
/// determine their values. The existing planner is not consulted for capability or schema approval.
///
/// The route is offered a whole expression because the compiler folds a range function and the math
/// functions over it into one tree: offering a sub-expression would compile a tree that computes
@@ -606,6 +613,10 @@ impl PromPlanner {
if !self.native_fused {
return Ok(None);
}
// The expression names the value columns of every rows layout this attempt publishes, and the
// node's own label follows the same text. It is the whole statement's expression even when a
// subquery root compiles its inner one, because that is the statement the reader wrote.
let expression = prom_expr.to_string();
// A range selector returns raw samples, which have no evaluation grid to bind. A subquery root
// instead returns its inner expression on the subquery's child grid, and a range query has no grid
// to bind a matrix root to.
@@ -689,7 +700,13 @@ impl PromPlanner {
let Some(values) = native_root_fields(tree.root()) else {
return Ok(None);
};
let time_index = if values
// A label named `time` is the one collision the default time column can have: the
// value columns are named after the expression, so they never take that name. The
// time column is this route's own synthetic name - no label or value is renamed - so
// a statement whose output already holds a `time` column publishes its grid under the
// first free `native_time_N` instead of dropping or renaming the label.
let value_names = expression_field_names(&expression, &values);
let time_index = if value_names
.iter()
.chain(&output_tags)
.any(|name| name == DEFAULT_TIME_INDEX_COLUMN)
@@ -697,7 +714,7 @@ impl PromPlanner {
(0usize..)
.map(|index| format!("native_time_{index}"))
.find(|candidate| {
!values
!value_names
.iter()
.chain(&output_tags)
.any(|name| name == candidate)
@@ -717,6 +734,7 @@ impl PromPlanner {
// A data-free tree reads no input, so every field it answers is a float.
let output_schema = match format {
FusedOutputFormat::PlainRows => fused_output_schema(
&expression,
&values,
&with_row_name_column(tree.root(), output_tags.clone()),
&time_index,
@@ -732,7 +750,7 @@ impl PromPlanner {
}
};
let node = FusedSeries::try_new_with_format(
Arc::from(prom_expr.to_string()),
Arc::from(expression.as_str()),
tree,
FusedSeriesBinding {
grid: Arc::clone(&grid),
@@ -1466,7 +1484,7 @@ impl PromPlanner {
if selectors.len() == 1
&& matches!(&tree.root().expr, NativeExpr::Merge { input } if pointwise_operand(input) && canonical_local_prefix(input) && !contains_global_consumer(input))
&& let Some((plan, _)) = canonical_boundary_stages(
Arc::from(prom_expr.to_string()),
Arc::from(expression.as_str()),
&tree,
&children,
&source_tags,
@@ -1477,6 +1495,7 @@ impl PromPlanner {
format,
root_kind,
fused_output_tags,
expression_field_names,
)
{
return Ok(Some(native_planned(
@@ -1510,7 +1529,7 @@ impl PromPlanner {
// caller asked for: it names one column per value the partial state accumulates,
// while the encoded frame carries those values inside its own layout.
let Some(value_schema) =
Self::fused_root_schema(&tree, &source_tags, &time, root_kind)
Self::fused_root_schema(&expression, &tree, &source_tags, &time, root_kind)?
else {
return Ok(None);
};
@@ -1525,7 +1544,7 @@ impl PromPlanner {
};
let publish_grid = Arc::clone(&grid);
let node = plan_split_aggregate(
Arc::from(prom_expr.to_string()),
Arc::from(expression.as_str()),
partial,
final_tree,
FusedSeriesBinding {
@@ -1556,7 +1575,7 @@ impl PromPlanner {
if required_placement || matches!(locality, InputLocality::SinglePartition) {
let segment_metric = if cross_table { None } else { metric.clone() };
let plan = segment_tree(
Arc::from(prom_expr.to_string()),
Arc::from(expression.as_str()),
&tree,
&children,
&source_tags,
@@ -1575,7 +1594,8 @@ impl PromPlanner {
// that is not a tag, and a partitioned table whose occurrences are source-backed.
any_split_series || (per_source_inputs && any_partitioned),
fused_output_tags,
PromPlanner::fused_root_schema,
expression_field_names,
PromPlanner::segmented_root_schema,
);
if let Some(plan) = plan {
return Ok(Some(native_planned(
@@ -1609,20 +1629,9 @@ impl PromPlanner {
let Some(root_fields) = native_root_fields(tree.root()) else {
return Ok(None);
};
// A root field can share a name with a tag or the time index, and under `or` the right
// table's tags join the left table's columns. The output names each column once, so such a
// statement keeps the existing plan.
let mut column_names = std::collections::HashSet::new();
if !std::iter::once(&time)
.chain(&root_fields)
.chain(&output_tags)
.all(|name| column_names.insert(name.as_str()))
{
return Ok(None);
}
let output_schema = match format {
FusedOutputFormat::PlainRows => {
fused_output_schema(&root_fields, &output_tags, &time, root_kind)?
fused_output_schema(&expression, &root_fields, &output_tags, &time, root_kind)?
}
FusedOutputFormat::SeriesFrame => {
let frame =
@@ -1645,7 +1654,7 @@ impl PromPlanner {
// binding is that grid rather than a per-shape choice.
let publish_grid = Arc::clone(&grid);
let node = FusedSeries::try_new_with_format(
Arc::from(prom_expr.to_string()),
Arc::from(expression.as_str()),
tree,
FusedSeriesBinding {
grid: publish_grid,
@@ -1673,36 +1682,56 @@ impl PromPlanner {
/// The columns a [`FusedSeries`] node over `tree` produces.
///
/// Field and tag names are derived from the native tree; a walk that cannot derive a tag set declines
/// rather than consulting the legacy planner.
/// Fields and tags are derived from the native tree; the expression names the output fields.
/// If this walk cannot derive a tag set, the native route declines rather than consulting the
/// legacy planner.
///
/// `None` when the root is not a value the node can publish, or when a field would share a name with a
/// tag or the time index.
/// `Ok(None)` when the root is not a value the node can publish. A schema the output cannot
/// declare is an error rather than a decline: duplicate names violate the expression-named
/// native output contract, so construction reports the collision instead of publishing a
/// layout the reader cannot address.
fn fused_root_schema(
expression: &str,
tree: &NativeTree,
source_tags: &[Vec<String>],
time: &str,
root_kind: FieldKind,
) -> Result<Option<DFSchemaRef>> {
let output_tags = match &tree.root().output {
NativeType::Scalar(_) => Vec::new(),
NativeType::Samples(_) => {
let Some(tags) = fused_output_tags(tree.root(), source_tags) else {
return Ok(None);
};
with_row_name_column(tree.root(), tags)
}
NativeType::AggregateState(_) => return Ok(None),
};
let Some(root_fields) = native_root_fields(tree.root()) else {
return Ok(None);
};
// The kind is the one the statement's own root answers, derived before segmentation cut
// anything. It cannot be recomputed here: a cut prefix is replaced by an input leaf whose
// samples are already evaluated, so walking this tree would read every such leaf as the
// kind its scan carried. `count_over_time(h[5m]) + on (__name__) ...` answers floats, and
// its replaced leaf carries floats, while the scan below was a histogram.
fused_output_schema(expression, &root_fields, &output_tags, time, root_kind).map(Some)
}
/// The rows layout a segmented statement root publishes, under the segment builders' callback.
///
/// A segment cannot carry this crate's error type, so a layout it cannot declare is a decline
/// here: the route keeps the plan the existing planner would have built for the same statement.
fn segmented_root_schema(
expression: &str,
tree: &NativeTree,
source_tags: &[Vec<String>],
time: &str,
root_kind: FieldKind,
) -> Option<DFSchemaRef> {
let output_tags = match &tree.root().output {
NativeType::Scalar(_) => Vec::new(),
NativeType::Samples(_) => {
with_row_name_column(tree.root(), fused_output_tags(tree.root(), source_tags)?)
}
NativeType::AggregateState(_) => return None,
};
let root_fields = native_root_fields(tree.root())?;
let mut column_names = std::collections::HashSet::new();
let columns = std::iter::once(time)
.chain(root_fields.iter().map(String::as_str))
.chain(output_tags.iter().map(String::as_str));
if !columns.clone().all(|name| column_names.insert(name)) {
return None;
}
// The kind is the one the statement's own root answers, derived before segmentation cut anything.
// It cannot be recomputed here: a cut prefix is replaced by an input leaf whose samples are
// already evaluated.
fused_output_schema(&root_fields, &output_tags, time, root_kind).ok()
Self::fused_root_schema(expression, tree, source_tags, time, root_kind)
.ok()
.flatten()
}
/// The statement being planned, rebuilt from the evaluation numbers the planner holds.
@@ -1821,18 +1850,67 @@ fn native_root_fields(node: &NativeNode) -> Option<Vec<String>> {
}
}
/// The output schema of a fused node: its time index, one column per field, then its tags.
/// The names an expression gives the value columns of `fields`.
///
/// The columns themselves are named by [`segment::fused_output_schema`], which is where a segment
/// asks for the same layout; this is that answer under this crate's error type, which is what the
/// planner's own call sites report.
/// One value is named after the whole expression; several append their compiled field names to it,
/// because one name cannot stand for more than one column. Only the published layout is named this
/// way: the fields inside the native tree keep the names the compiler gave them.
fn expression_field_names(expression: &str, fields: &[String]) -> Vec<String> {
fields
.iter()
.map(|field| {
if fields.len() == 1 {
expression.to_string()
} else {
format!("{expression}.{field}")
}
})
.collect()
}
/// The columns a [`FusedSeries`] node produces: its time index, one column per value, then its tags.
///
/// A single value is named after the expression, and multiple values append their compiled field
/// names to it, so a value is never published under a name the reader cannot address. The names
/// inside the native tree are unchanged.
///
/// The layout itself is [`segment::fused_output_schema`]'s, which is where a segment asks for the
/// same columns; this is that answer under this crate's error type, which is what the planner's own
/// call sites report.
fn fused_output_schema(
expression: &str,
fields: &[String],
tags: &[String],
time_index: &str,
field_kind: FieldKind,
) -> Result<DFSchemaRef> {
segment::fused_output_schema(fields, tags, time_index, field_kind)
let value_names = expression_field_names(expression, fields);
// A name appears twice when a value named after its expression - a bare selector is named after
// its metric - lands on the table's own time index or on one of its tags, or when two values
// take one name. `DFSchema` rejects that, but reports only that building the plan failed, which
// names no column; the collision is detected here so the error names the columns that collide.
let mut names = Vec::with_capacity(value_names.len() + tags.len() + 1);
names.push(time_index.to_string());
names.extend(value_names.iter().cloned());
names.extend(tags.iter().cloned());
let mut occurrences: std::collections::HashMap<&str, usize> =
std::collections::HashMap::with_capacity(names.len());
for name in &names {
*occurrences.entry(name.as_str()).or_default() += 1;
}
let mut reported = std::collections::HashSet::new();
let duplicates = names
.iter()
.filter(|name| occurrences[name.as_str()] > 1 && reported.insert(name.as_str()))
.map(String::as_str)
.collect::<Vec<_>>();
ensure!(
duplicates.is_empty(),
DuplicateOutputColumnsSnafu {
names: duplicates.join(", ")
}
);
segment::fused_output_schema(&value_names, tags, time_index, field_kind)
.context(DataFusionPlanningSnafu)
}
@@ -2911,8 +2989,10 @@ mod tests {
("x or y", two()),
] {
let tree = compiled_test_tree(query);
let schema = PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float)
.unwrap_or_else(|| panic!("{query}: a samples root declares a rows layout"));
let schema =
PromPlanner::fused_root_schema(query, &tree, &sources, "ts", FieldKind::Float)
.unwrap()
.unwrap_or_else(|| panic!("{query}: a samples root declares a rows layout"));
let names = schema
.fields()
.iter()
@@ -2932,9 +3012,13 @@ mod tests {
assert_eq!(ArrowDataType::Utf8, *name.data_type(), "{query}");
}
let scalar = compiled_test_tree("1 + 2");
let schema = PromPlanner::fused_root_schema(&scalar, &[], "ts", FieldKind::Float).unwrap();
let schema = PromPlanner::fused_root_schema("1 + 2", &scalar, &[], "ts", FieldKind::Float)
.unwrap()
.unwrap();
assert_eq!(
vec!["ts", "value"],
// A scalar root declares no name column, and its one value is named after the
// expression, as every other rows layout names its values.
vec!["ts", "1 + 2"],
schema
.fields()
.iter()
@@ -2948,10 +3032,11 @@ mod tests {
fn a_present_name_tag_is_not_declared_twice() {
let sources = [vec!["host".to_string(), METRIC_NAME.to_string()]];
let tree = compiled_test_tree("x");
let schema =
PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float).unwrap();
let schema = PromPlanner::fused_root_schema("x", &tree, &sources, "ts", FieldKind::Float)
.unwrap()
.unwrap();
assert_eq!(
vec!["ts", "v0", "host", METRIC_NAME],
vec!["ts", "x", "host", METRIC_NAME],
schema
.fields()
.iter()
@@ -2960,19 +3045,34 @@ mod tests {
);
}
/// A value column that is itself named `__name__` is a column of the result, not the name of a
/// row: the layout names each column once, so such a statement keeps the existing plan.
/// The value columns are named after the expression, so a tree whose own field is named
/// `__name__` still publishes that value under the expression's name - the name column is the
/// layout's own. A time index of that name collides with it, and the collision is reported
/// rather than published as a layout the reader cannot address.
#[test]
fn a_column_named_after_the_metric_name_declines_the_rows_layout() {
fn a_time_index_named_after_the_metric_name_is_reported() {
let sources = [vec!["host".to_string()]];
// The field is renamed by the expression, so it no longer stands where the name column is.
let tree = compiled_test_tree_with_fields("x", &[METRIC_NAME]);
assert!(PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float).is_none());
// A time index of that name collides the same way.
let tree = compiled_test_tree("x");
assert!(
PromPlanner::fused_root_schema(&tree, &sources, METRIC_NAME, FieldKind::Float)
.is_none()
let schema = PromPlanner::fused_root_schema("x", &tree, &sources, "ts", FieldKind::Float)
.unwrap()
.unwrap_or_else(|| panic!("the value is named after the expression"));
assert_eq!(
vec!["ts", "x", "host", METRIC_NAME],
schema
.fields()
.iter()
.map(|field| field.name().as_str())
.collect::<Vec<_>>()
);
// A time index of that name collides with the row-name column, and the error names it.
let tree = compiled_test_tree("x");
let error =
PromPlanner::fused_root_schema("x", &tree, &sources, METRIC_NAME, FieldKind::Float)
.expect_err("a time index named after the metric collides with the name column");
let message = error.to_string().to_lowercase();
assert!(message.contains("duplicate"), "{message}");
assert!(message.contains(METRIC_NAME), "{message}");
}
/// A `__name__` rewrite publishes a name column, canonicalizes the complete runtime label set,
@@ -3145,7 +3245,9 @@ mod tests {
.map(|field| field.name.as_ref())
.collect::<Vec<_>>()
);
let expression = "sum by (zone, host) ((x))";
let output = fused_output_schema(
expression,
&["v0".to_string(), "v1".to_string()],
&["zone".to_string(), "host".to_string()],
"ts",
@@ -3153,7 +3255,13 @@ mod tests {
)
.unwrap();
assert_eq!(
vec!["ts", "v0", "v1", "zone", "host"],
vec![
"ts".to_string(),
format!("{expression}.v0"),
format!("{expression}.v1"),
"zone".to_string(),
"host".to_string(),
],
output
.fields()
.iter()
File diff suppressed because it is too large Load Diff
@@ -87,6 +87,7 @@ use promql::vector_match::MatchingLabels;
use query::datafusion::QUERY_PARALLELISM_HINT;
use query::dist_plan::MergeScanExec;
use query::options::QueryOptions;
use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
use query::physical_wrapper::{PhysicalPlanWrapper, PhysicalPlanWrapperRef};
use servers::query_handler::sql::SqlQueryHandler;
use session::context::{QueryContext, QueryContextRef};
@@ -430,9 +431,6 @@ struct AggregateCase {
partial_state: Option<&'static str>,
/// Whether this case's global aggregate has an explicitly specified single owner.
single_owner: bool,
/// Native output schema. The native aggregate preserves its source field name (`v`), rather
/// than the legacy planner's computed-expression name.
native_value_column: &'static str,
}
/// Builds one case from a PromQL expression and its `(labels, value)` rows.
@@ -454,7 +452,6 @@ fn case(promql: String, expected: &[(&[(&str, &str)], f64)], per_partition: bool
root_values: None,
partial_state: None,
single_owner: false,
native_value_column: "v",
}
}
@@ -485,6 +482,19 @@ fn tql_eval(promql: &str) -> String {
format!("TQL EVAL ({EVAL_SECS}, {EVAL_SECS}, '1m') {promql}")
}
fn expression_name(promql: &str) -> String {
let query = PromQuery {
query: promql.to_string(),
..PromQuery::default()
};
let QueryStatement::Promql(stmt, _) =
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
else {
unreachable!()
};
stmt.expr.to_string()
}
/// Runs `promql` at [`EVAL_SECS`] and returns its rows with the native nodes its plans held.
async fn run_query(
instance: &Arc<Instance>,
@@ -1192,12 +1202,12 @@ fn assert_rows(actual: &[ResultRow], expected: &[ResultRow], context: &str) {
}
/// Asserts the native boundary's ordered names and types, independently of legacy names.
fn assert_native_column_layout(native: &QueryResult, tags: &[&str], query: &str) {
let mut expected = vec!["ts", "v"];
expected.extend_from_slice(tags);
fn assert_native_column_layout(native: &QueryResult, tags: &[&str], promql: &str, query: &str) {
let mut expected = vec!["ts".to_string(), expression_name(promql)];
expected.extend(tags.iter().map(|tag| (*tag).to_string()));
// The rows layout declares the per-row name column after the labels; an aggregate computes a
// value of its own, so that column is null here.
expected.push("__name__");
expected.push("__name__".to_string());
assert_eq!(
expected, native.columns,
"`{query}`: native boundary columns"
@@ -1281,7 +1291,7 @@ fn assert_native_columns(
);
assert_eq!(
expected_value_column, native.value_column,
"`{query}`: native value column must use the compiled root field name"
"`{query}`: native value column must use the statement expression"
);
assert_eq!(
1,
@@ -1774,7 +1784,7 @@ async fn check_two_source_segregation(
legacy.columns,
"`{query}`: the legacy plan names the value column after its expression"
);
assert_native_column_layout(&native, &[], &query);
assert_native_column_layout(&native, &[], promql, &query);
assert_rows(
&native.rows,
&legacy.rows,
@@ -1883,7 +1893,7 @@ async fn check_nested_aggregate_case(
assert_rows(&legacy.rows, &case.expected, &format!("legacy `{query}`"));
let (native, capture) = run_query(instance, &case.promql, true, observer).await;
assert_native_column_layout(&native, &[], &query);
assert_native_column_layout(&native, &[], &case.promql, &query);
assert_rows(
&native.rows,
&legacy.rows,
@@ -1916,7 +1926,7 @@ async fn check_case(instance: &Arc<Instance>, observer: &NativePlanObserver, cas
.keys()
.map(String::as_str)
.collect::<Vec<_>>();
assert_native_column_layout(&native, &tags, &query);
assert_native_column_layout(&native, &tags, promql, &query);
if promql.starts_with("count ") {
assert!(
legacy.column_types.contains(&DataType::Int64),
@@ -1928,8 +1938,8 @@ async fn check_case(instance: &Arc<Instance>, observer: &NativePlanObserver, cas
&legacy.rows,
&format!("native `{query}` vs legacy"),
);
assert_native_schema(&native, case.native_value_column, &case.expected, &query);
assert_native_columns(&native, &legacy, &query, "v");
assert_native_schema(&native, &expression_name(promql), &case.expected, &query);
assert_native_columns(&native, &legacy, &query, &expression_name(promql));
assert_rows(&native.rows, &case.expected, &format!("native `{query}`"));
assert_eq!(
@@ -2190,7 +2200,7 @@ async fn native_aggregate_topk_over_global_partitions() {
let (legacy, legacy_capture) = run(false).await;
assert!(legacy_capture.fused.is_empty() && legacy_capture.finalize.is_empty());
let (native, capture) = run(true).await;
assert_native_columns(&native, &legacy, &sql, "v");
assert_native_columns(&native, &legacy, &sql, &expression_name(&promql));
assert_rows(
&native.rows,
&legacy.rows,
@@ -2225,7 +2235,7 @@ async fn native_aggregate_topk_over_global_partitions() {
}
}
expected.sort_by(|a, b| (a.ts_millis, &a.labels).cmp(&(b.ts_millis, &b.labels)));
assert_native_schema(&native, "v", &expected, &sql);
assert_native_schema(&native, &expression_name(&promql), &expected, &sql);
assert_rows(&native.rows, &expected, &format!("expected `{sql}`"));
// `topk` above a global aggregate is not split, so the scan side folds per series and the
// ranking runs above it. A plan may hold several fused nodes (the frontend root and the region
@@ -2295,9 +2305,13 @@ async fn native_aggregate_clamp_executes_on_both_routes() {
close_enough(native.rows[0].value.unwrap(), 10.0),
"native `{query}` value"
);
assert_eq!("v", native.value_column, "native `{query}` value column");
// The rows layout: the time index, the one value, and the declared name column, which an
// aggregate leaves null.
assert_eq!(
expression_name(&promql),
native.value_column,
"native `{query}` value column"
);
// The rows layout: the time index, the one expression-named value, and the declared name
// column, which an aggregate leaves null.
assert_eq!(3, native.columns.len(), "native `{query}` columns");
assert_eq!("__name__", native.columns[2], "native `{query}` columns");
@@ -2781,9 +2795,9 @@ async fn native_histogram_over_the_cut_splits_and_matches_legacy() {
// reports over the complete input, labels included - is compared before the plan shape, so a
// shape regression cannot hide it.
let tags: &[&str] = if *single_owner { &[] } else { &["job"] };
assert_native_column_layout(&native, tags, &query);
assert_native_schema(&native, "v", &legacy.rows, &query);
assert_native_columns(&native, &legacy, &query, "v");
assert_native_column_layout(&native, tags, promql, &query);
assert_native_schema(&native, &expression_name(promql), &legacy.rows, &query);
assert_native_columns(&native, &legacy, &query, &expression_name(promql));
assert_rows(
&native.rows,
&legacy.rows,
@@ -3032,15 +3046,19 @@ async fn native_bare_selector_aggregate_value_columns() {
let mark = observer.mark();
let (native_names, native_types, native_rows, capture) =
read(&instance, &observer, &promql, true).await;
let expression = expression_name(&promql);
let mut expected_names = vec!["ts".to_string()];
match expected_values {
1 => expected_names.push(expression),
2 => expected_names.extend([format!("{expression}.a"), format!("{expression}.b")]),
_ => unreachable!(),
}
if !single_owner {
expected_names.push("job".to_string());
}
// The rows layout declares the per-row name column after the labels; every statement in
// this case is an aggregate, which computes its value and leaves that column null.
let expected_names = match (expected_values, single_owner) {
(1, true) => vec!["ts", "v", "__name__"],
(1, false) => vec!["ts", "v", "job", "__name__"],
(2, true) => vec!["ts", "a", "b", "__name__"],
(2, false) => vec!["ts", "a", "b", "job", "__name__"],
_ => unreachable!(),
};
expected_names.push("__name__".to_string());
assert_eq!(
expected_names, native_names,
"`{query}` native boundary columns"
+346 -97
View File
@@ -5256,11 +5256,15 @@ async fn distributed_native_promql_zero_match_or_real() {
);
let (columns, rows) = native_labelled_rows(output).await;
assert_eq!(
columns,
vec!["__name__", "host", "ts", "val"],
"{expression}"
);
// The value is named after the statement's expression, not `val`.
let mut expected_columns = vec![
"__name__".to_string(),
"host".to_string(),
"ts".to_string(),
expression_name(expression),
];
expected_columns.sort();
assert_eq!(columns, expected_columns, "{expression}");
let actual = rows
.iter()
.map(|(time, labels, value)| {
@@ -5454,7 +5458,7 @@ async fn distributed_native_promql_metric_discovery_reconciles_mixed_ordering()
.clone()
});
let values = batch
.column_by_name("val")
.column_by_name(&expression_name(expression))
.expect("one value column")
.as_any()
.downcast_ref::<Float64Array>()
@@ -5626,9 +5630,7 @@ async fn data_free_rows(output: Output) -> (Vec<String>, Vec<(i64, f64)>) {
(columns, rows)
}
/// A statement that reads no table plans natively over the wire, where there is no region to place
/// anything in: the frontend evaluates one fused node, so the answer keeps the native column name
/// instead of the one the existing planner derives from an expression.
/// A statement that reads no table plans natively over the wire, with an expression-named value.
#[tokio::test(flavor = "multi_thread")]
async fn distributed_native_promql_data_free_statement_plans_natively() {
common_telemetry::init_default_ut_logging();
@@ -5644,15 +5646,15 @@ async fn distributed_native_promql_data_free_statement_plans_natively() {
.build(false)
.await;
let instance = cluster.fe_instance().clone();
for (expression, expected_columns, expected_rows) in [
for (expression, labels, expected_rows) in [
// `start`/`end` are seconds, so this one-step range evaluates at 3000ms.
("time()", &["time", "value"][..], &[(3000, 3.0)][..]),
("time()", &[][..], &[(3000, 3.0)][..]),
(
// A samples root declares the name of each row it publishes, and `vector(1)` has
// none, so the column is present and null. `time()` above is a scalar root and keeps
// two columns.
r#"label_replace(vector(1), "host", "host1", "", "")"#,
&["time", "value", "host", "__name__"][..],
&["host", "__name__"][..],
&[(3000, 1.0)][..],
),
] {
@@ -5679,11 +5681,17 @@ async fn distributed_native_promql_data_free_statement_plans_natively() {
"a data-free statement reads no region: {plan:?}"
);
let (columns, rows) = data_free_rows(output).await;
let QueryStatement::Promql(stmt, _) =
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
else {
unreachable!()
};
let mut expected_columns = vec!["time".to_string(), stmt.expr.to_string()];
expected_columns.extend(labels.iter().map(|label| (*label).to_string()));
assert_eq!(expected_columns, columns, "{expression}");
assert_eq!(expected_rows, rows.as_slice(), "{expression}");
// The same statement through the existing planner answers the same rows; only the native
// route names the value column after the tree rather than after an expression.
// The same statement through the existing planner answers the same rows.
let legacy = instance
.do_promql_query(&query, native_query_context(false, 2))
.await
@@ -6805,9 +6813,20 @@ async fn native_promql_sort_absent_and_histogram_quantile_match_legacy() {
let legacy_rows = legacy.iter().map(|batch| batch.num_rows()).sum::<usize>();
assert_eq!(native_rows, legacy_rows);
assert!(native_rows >= 1, "histogram_quantile must produce data");
// The rows root lays its columns out as time, values, labels, and the name last.
let query = PromQuery {
query: shape.to_string(),
..PromQuery::default()
};
let QueryStatement::Promql(stmt, _) =
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
else {
unreachable!()
};
let expression = stmt.expr.to_string();
// The rows root lays its columns out as time, the expression-named value, the labels,
// and the name last.
assert_eq!(
vec!["ts", "val", "host", "__name__"],
vec!["ts", expression.as_str(), "host", "__name__"],
native_columns
.iter()
.map(|column| column.name.as_str())
@@ -6902,15 +6921,132 @@ async fn native_promql_sort_absent_and_histogram_quantile_match_legacy() {
// keep the metric name; `absent` answers a value of its own for a label set it derives
// from the matcher, and Prometheus' `absent` leaves the name out of that set. The
// existing route has no name column at all, so the column is checked against what the
// shape answers and left out of the comparison.
// shape answers and left out of the comparison below.
//
// Both routes answer the same rows; only the one value column's header differs,
// because the existing planner keeps the tree's field name while the native route
// names that column after the expression. Compare what the columns carry - the
// timestamp, the label set and the value, in response order - and the native naming.
let reported = if shape.starts_with("absent") {
ReportedName::Nothing
} else {
ReportedName::Metric("native_shapes")
};
let native = named_rows_text(native, reported).await;
let legacy = named_rows_text(legacy, ReportedName::NoColumn).await;
assert_eq!(native, legacy, "{shape}; SQL: {sql}");
let collect = |output: Output| async move {
match output.data {
OutputData::Stream(stream) => collect_batches(stream).await.unwrap(),
OutputData::RecordBatches(batches) => batches,
_ => unreachable!("query returns record batches"),
}
};
let rows = |batches: &common_recordbatch::RecordBatches, reported: ReportedName<'_>| {
batches
.iter()
.flat_map(|batch| {
let batch = batch.df_record_batch();
let schema = batch.schema();
// The name column is checked against what the shape answers and kept out of
// the label set, so the rows of the two routes compare whole.
checked_name_column(schema.fields(), batch.columns(), reported);
let value_index = schema
.fields()
.iter()
.position(|field| field.data_type() == &DataType::Float64)
.expect("one float value column");
let timestamps = batch
.column_by_name("ts")
.unwrap()
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.unwrap();
let labels = schema
.fields()
.iter()
.filter(|field| {
!matches!(
field.data_type(),
DataType::Float64 | DataType::Timestamp(..)
) && field.name() != "__name__"
})
.map(|field| {
let column = cast(
batch.column_by_name(field.name()).unwrap().as_ref(),
&DataType::Utf8,
)
.unwrap();
(
field.name().clone(),
column
.as_any()
.downcast_ref::<StringArray>()
.unwrap()
.clone(),
)
})
.collect::<Vec<_>>();
let values =
cast(batch.column(value_index).as_ref(), &DataType::Float64).unwrap();
let values = values.as_any().downcast_ref::<Float64Array>().unwrap();
(0..batch.num_rows())
.map(|row| {
(
timestamps.value(row),
labels
.iter()
.filter(|(_, column)| {
column.is_valid(row) && !column.value(row).is_empty()
})
.map(|(name, column)| {
(name.clone(), column.value(row).to_string())
})
.collect::<std::collections::BTreeMap<_, _>>(),
values.value(row),
)
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>()
};
let native = collect(native).await;
let legacy = collect(legacy).await;
let native_names = native
.schema()
.column_schemas()
.iter()
.map(|column| column.name.clone())
.collect::<Vec<_>>();
let native_rows = rows(&native, reported);
let legacy_rows = rows(&legacy, ReportedName::NoColumn);
assert_eq!(native_rows, legacy_rows, "{shape}; SQL: {sql}");
if shape.starts_with("sort") {
assert!(!native_rows.is_empty(), "{shape} must produce data: {sql}");
}
let legacy_labels = legacy
.schema()
.column_schemas()
.iter()
.filter(|column| {
!column.data_type.is_timestamp()
&& column.data_type
!= datatypes::prelude::ConcreteDataType::float64_datatype()
})
.map(|column| column.name.clone())
.collect::<Vec<_>>();
// The native answer's columns are the time index, the expression-named value, the
// labels the existing route also declares, and the name column it appends last.
let mut expected_names = vec!["ts".to_string(), expression_name(shape)];
expected_names.extend(legacy_labels);
expected_names.push("__name__".to_string());
assert_eq!(
native_names, expected_names,
"{shape}: native boundary columns"
);
assert!(
native_names[2..native_names.len() - 1]
.iter()
.all(|name| matches!(name.as_str(), "host" | "le")),
"{shape}: native boundary columns: {native_names:?}"
);
}
}
}
@@ -7541,11 +7677,10 @@ async fn native_kind_rows(output: Output) -> Vec<(i64, String, String, f64)> {
let hosts = hosts.as_any().downcast_ref::<StringArray>().unwrap();
let kinds = kinds.as_any().downcast_ref::<StringArray>().unwrap();
let values = batch
.column_by_name("val")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
.columns()
.iter()
.find_map(|column| column.as_any().downcast_ref::<Float64Array>())
.expect("one value column");
for row in 0..batch.num_rows() {
rows.push((
timestamps.value(row),
@@ -8157,8 +8292,15 @@ async fn native_promql_zero_match_or_real_tql() {
// A rows answer reports the statement's own columns, the per-row name column among them.
// `alpha_total` reads one named metric, and `or` reports each side with the label set it
// read, so every row here is a series of that metric and keeps its name while the empty
// side adds nothing.
assert_eq!(columns, vec!["__name__", "host", "ts", "val"], "{query}");
// side adds nothing. The value is named after the statement's expression, not `val`.
let mut expected_columns = vec![
"__name__".to_string(),
"host".to_string(),
"ts".to_string(),
expression_name(query),
];
expected_columns.sort();
assert_eq!(columns, expected_columns, "{query}");
let actual = rows
.iter()
.map(|(time, labels, value)| {
@@ -8499,6 +8641,19 @@ async fn discovery_query(instance: &Arc<Instance>, promql: &str) -> Output {
.unwrap()
}
fn expression_name(promql: &str) -> String {
let query = PromQuery {
query: promql.to_string(),
..PromQuery::default()
};
let QueryStatement::Promql(stmt, _) =
QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap()
else {
unreachable!()
};
stmt.expr.to_string()
}
/// `(time, labels, value)` of every row, sorted, whatever the columns are named: the time index is
/// the timestamp column, the value the one `Float64` column, and every other column a label that a
/// NULL or empty value leaves out. Also returns the column names.
@@ -8860,7 +9015,18 @@ async fn native_promql_cross_table_pairs_over_different_schemas() {
2,
"{context}: expected one distinct SeriesScanExec per selector occurrence"
);
assert_eq!(native_columns, columns, "{context}");
let mut expected_columns = columns
.into_iter()
.map(|column| {
if column == "val" {
expression_name(query)
} else {
column.to_string()
}
})
.collect::<Vec<_>>();
expected_columns.sort();
assert_eq!(native_columns, expected_columns, "{context}");
assert_eq!(native.len(), expected.len(), "{context}: {native:?}");
for (actual, expected) in native.iter().zip(&expected) {
assert_eq!(
@@ -9055,7 +9221,27 @@ async fn native_promql_three_selector_trees_bind_every_occurrence() {
.map(|field| field.name().clone())
.collect::<Vec<_>>();
plan_columns.sort();
assert_eq!(plan_columns, columns, "{query}");
let expression_query = PromQuery {
query: query.to_string(),
..PromQuery::default()
};
let QueryStatement::Promql(stmt, _) =
QueryLanguageParser::parse_promql(&expression_query, &QueryContext::arc()).unwrap()
else {
unreachable!()
};
let mut expected_columns = columns
.into_iter()
.map(|column| {
if column == "val" {
stmt.expr.to_string()
} else {
column.to_string()
}
})
.collect::<Vec<_>>();
expected_columns.sort();
assert_eq!(plan_columns, expected_columns, "{query}");
assert_eq!(native, expected, "{query}");
let legacy = native_result_rows(native_tql_output(&instance, &sql, false).await).await;
assert_eq!(native, legacy, "legacy: {query}");
@@ -9278,7 +9464,18 @@ async fn native_promql_info_enriches_from_the_info_table() {
assert_eq!(plan.name(), "FusedSeriesExec", "{query}");
assert_eq!(plan.children().len(), 2, "{query}");
assert_eq!(observer.take_raw_sources().len(), 2, "{query}");
assert_eq!(native_columns, columns, "{query}");
let mut expected_columns = columns
.into_iter()
.map(|column| {
if column == "val" {
expression_name(query)
} else {
column.to_string()
}
})
.collect::<Vec<_>>();
expected_columns.sort();
assert_eq!(native_columns, expected_columns, "{query}");
assert_eq!(native, expected, "{query}");
// The existing plan refuses `info`: "Unsupported expr type: info" for one argument, and
@@ -10085,10 +10282,18 @@ async fn native_hybrid_binary_empty_on_grouped_matching_and_errors() {
native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
let plan = observer.two_input_fused();
assert_eq!(plan.name(), "FusedSeriesExec", "{query}");
assert_eq!(
columns,
vec!["__name__", "host", "kind", "ts", "val", "zone_name"]
);
// The value is named after the statement's expression, and the rows answer reports the
// per-row name column among its labels; a math kernel drops the name, so it holds no value.
let mut expected_columns = vec![
"__name__".to_string(),
"host".to_string(),
"kind".to_string(),
"ts".to_string(),
expression_name(query),
"zone_name".to_string(),
];
expected_columns.sort();
assert_eq!(columns, expected_columns);
assert_eq!(rows, expected, "{query}");
let (_, legacy) =
native_labelled_rows(native_tql_output(&instance, &sql, false).await).await;
@@ -10582,6 +10787,26 @@ async fn native_promql_metric_two_aggregate_pair_matches_legacy() {
.iter()
.map(|column| column.name.as_str())
.collect::<Vec<_>>();
let expression_query = PromQuery {
query: query.to_string(),
..PromQuery::default()
};
let QueryStatement::Promql(stmt, _) =
QueryLanguageParser::parse_promql(&expression_query, &QueryContext::arc()).unwrap()
else {
unreachable!()
};
let expression = stmt.expr.to_string();
let expected_columns = expected_columns
.into_iter()
.map(|column| {
if column == "val" {
expression.as_str()
} else {
column
}
})
.collect::<Vec<_>>();
assert_eq!(
expected_columns, boundary_columns,
"{query}: native boundary columns"
@@ -10657,15 +10882,13 @@ async fn native_promql_metric_two_aggregate_pair_matches_legacy() {
// The columns both routes report: the time index, one value column, and one column per
// label. The labels themselves are compared row by row above, and the time index is `ts` on
// both. The value column's *name* is not equal, and that is this route's existing naming for
// a computed value column rather than anything this shape introduces: the node names its
// fields after the tree's own root schema, while the existing planner names a computed column
// after its expression (the same difference is pinned for a cross-table pair by
// `native_fused_cross_table_pair_reads_each_table_through_its_own_source`). So the
// shape is asserted, and the value column's name is pinned as the raw field it comes from.
// The rows layout declares the per-row name column and this statement has none to report:
// a division computes a value of its own and its inputs are aggregates, so every row's name
// is null. The column is therefore the only difference from the existing route's schema.
// both. Compare time and label columns independently: the native boundary names that value
// column after the canonical expression, where the existing planner names a computed column
// after its expression too, and the two texts differ in the spaces a PromQL parse and a
// legacy rewrite produce. The rows layout additionally declares the per-row name column and
// this statement has none to report: a division computes a value of its own and its inputs
// are aggregates, so every row's name is null. That column is therefore the one difference
// from the existing route's schema.
assert_eq!(
native_columns.len(),
legacy_columns.len() + 1,
@@ -11710,13 +11933,19 @@ async fn native_promql_rewrites_the_label_count_values_writes() {
);
}
/// `topk`/`bottomk` key their groups by series labels, while the existing plan also accepts a value
/// field there and forms one group per sample value. A `by(<value field>)` therefore keeps every
/// group's winner, and the native route has to report the same rows: the union below it reports its
/// series, and the window above it partitions by the same value column. The rows are what this
/// asserts - one winner per value group - rather than that no native node answers the union.
/// `topk`/`bottomk` key their groups by series labels, and `by (val)` names no label a series
/// carries: `val` is the value field, not a label. Prometheus therefore puts every series of the
/// input into one empty group and answers only that group's winner, so the statement returns one
/// row rather than one per sample value.
///
/// Both routes are about that grouping, but only the native one answers Prometheus here. The value
/// column is named after the expression, so a `by (val)` over it selects nothing and the one empty
/// group keeps the global winner. The existing plan still partitions its window by the `val` column
/// it publishes, one group per sample value; that divergence is known and is not pinned here. What
/// the native route must keep is the answer, with the union below it running as one native node
/// rather than a legacy join.
#[tokio::test(flavor = "multi_thread")]
async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() {
async fn native_promql_ranks_a_value_field_grouping_as_one_group() {
common_telemetry::init_default_ut_logging();
let (standalone, observer) = native_test_instance(GreptimeDbStandaloneBuilder::new(
"native_promql_ranking_value_field",
@@ -11748,28 +11977,25 @@ async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() {
.await;
let pair = r#"native_rank_demo{host="x"} or native_rank_other{host="y"}"#;
// `by (val)` forms two groups of one sample each, so both series survive: two rows, one winner
// per value group, on both routes.
for query in [
format!("topk by (val) (1, {pair})"),
format!("bottomk by (val) (1, {pair})"),
// No series carries a `val` label, so the grouping is empty: every series lands in that one
// group and the answer is its winner, `y` at 8 for `topk` and `x` at 2 for `bottomk`.
for (query, expected) in [
(
format!("topk by (val) (1, {pair})"),
vec![(0, "y".to_string(), 8.0)],
),
(
format!("bottomk by (val) (1, {pair})"),
vec![(0, "x".to_string(), 2.0)],
),
] {
let sql = format!("TQL EVAL (0, 0, '1s') {query}");
observer.clear();
let legacy_rows = native_result_rows(native_tql_output(&instance, &sql, false).await).await;
assert_eq!(
legacy_rows,
vec![(0, "x".to_string(), 2.0), (0, "y".to_string(), 8.0)],
"{query}: the existing plan keeps one row per value group"
);
observer.clear();
let native_rows = native_result_rows(native_tql_output(&instance, &sql, true).await).await;
let root = datafusion::physical_plan::displayable(observer.root_plan().as_ref())
.indent(true)
.to_string();
// The union has to stay native rather than become the legacy join, and it must not fold the
// two value groups into one.
// The union has to stay native rather than become the legacy join.
assert!(
observer.fused().is_some(),
"{query}: the union runs as one fused node: {root}"
@@ -11778,7 +12004,7 @@ async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() {
!root.contains("JoinExec"),
"{query}: the union is not a legacy join: {root}"
);
assert_eq!(native_rows, legacy_rows, "{query}: full rows");
assert_eq!(native_rows, expected, "{query}: full rows");
}
// A label grouping is what the native node does honor, so it still runs natively and keeps every
@@ -12090,7 +12316,16 @@ async fn native_promql_delta_temporality_rate_and_increase_match_legacy() {
observer.clear();
let (columns, rows) =
native_labelled_rows(native_tql_output(&instance, &sql, true).await).await;
assert_eq!(columns, vec!["__name__", "host", "ts", "val"]);
// The value is named after the statement's expression, not `val`, and the rows answer
// reports the per-row name column among its labels; a division drops the name.
let mut expected_columns = vec![
"__name__".to_string(),
"host".to_string(),
"ts".to_string(),
expression_name(&query),
];
expected_columns.sort();
assert_eq!(columns, expected_columns, "{query}");
assert_eq!(rows.len(), 1, "{query}: {rows:?}");
assert_eq!(rows[0].0, 120_000, "{query}");
assert_eq!(
@@ -12792,9 +13027,10 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() {
// `count_values` names a label no scan carries, so the statement keeps a plan that reports it.
let sql = "TQL EVAL (0, 0, '1s') count_values(\"bucket\", vector(1))";
// The native route answers `count_values` itself now, so this is its own answer: one series at
// t=0 whose generated label is "1" and whose count is 1. The aggregate drops the metric name -
// as Prometheus does - so the declared name column is there and null, and it is the one column
// the existing planner's answer does not have; the two headers differ while the numbers agree.
// t=0 whose generated label is "1" and whose count is 1. The value column is named after the
// expression, and the aggregate drops the metric name - as Prometheus does - so the declared
// name column is there and null; it is the one column the existing planner's answer does not
// have, and is left out of the columns compared below.
let native = named_rows_text(
native_tql_output(&instance, sql, true).await,
ReportedName::Nothing,
@@ -12802,11 +13038,11 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() {
.await;
assert_eq!(
native,
"+---------------------+-------+--------+\
\n| time | value | bucket |\
\n+---------------------+-------+--------+\
\n| 1970-01-01T00:00:00 | 1.0 | 1 |\
\n+---------------------+-------+--------+"
"+---------------------+-----------------------------------+--------+\
\n| time | count_values(\"bucket\", vector(1)) | bucket |\
\n+---------------------+-----------------------------------+--------+\
\n| 1970-01-01T00:00:00 | 1.0 | 1 |\
\n+---------------------+-----------------------------------+--------+"
);
}
@@ -13618,10 +13854,16 @@ async fn native_promql_recursive_matching_segments_match_legacy() {
expected, native,
"matching on host drops job from output labels"
);
assert_eq!(
vec!["__name__", "host", "ts", "val"],
columns.iter().map(String::as_str).collect::<Vec<_>>()
);
// The value is named after the statement's expression, and the rows answer reports the
// per-row name column among its labels.
let mut expected_columns = vec![
"__name__".to_string(),
"host".to_string(),
"ts".to_string(),
expression_name(query),
];
expected_columns.sort();
assert_eq!(expected_columns, columns);
}
/// Segments are cut where the physical planner would otherwise have to gather a per-series prefix.
@@ -14199,18 +14441,17 @@ async fn native_promql_real_query_q07_throttled_periods() {
.iter()
.all(|debug| debug.contains("FusedSeries"))
);
// Known computed-column naming gap from the ungrouped two-source case; values agree.
// The two routes name the computed value column differently - the existing planner after its
// own rewrite, the native route after the expression - and the labelled rows compared above
// are what both report.
assert!(
legacy_columns
.iter()
.any(|column| column.contains("prom_increase"))
);
assert_eq!(
native_columns
.iter()
.filter(|column| *column == "val")
.count(),
1
assert!(
native_columns.contains(&expression_name(query)),
"Q07 native value column: {native_columns:?}"
);
}
@@ -14316,14 +14557,13 @@ async fn native_promql_real_query_q04_namespace_cpu_requests() {
.iter()
.all(|debug| debug.contains("FusedSeries"))
);
// Known computed-column naming gap from the ungrouped two-source case; values agree.
// The two routes name the computed value column differently - the existing planner after its
// own rewrite, the native route after the expression - and the labelled rows compared above
// are what both report.
assert!(legacy_columns.iter().any(|column| column.contains("max(")));
assert_eq!(
native_columns
.iter()
.filter(|column| *column == "val")
.count(),
1
assert!(
native_columns.contains(&expression_name(query)),
"Q04 native value column: {native_columns:?}"
);
}
@@ -14668,11 +14908,20 @@ async fn native_promql_dynamic_topk_matches_legacy() {
"a bare topk above no aggregate runs natively: {bare_sql}"
);
// The same ranking answers the same rows on both routes; the rows keep the name of the series
// `topk` selected, which this route states in its own declared column.
let mut named_columns = bare_legacy.0.clone();
named_columns.push("__name__".to_string());
named_columns.sort();
assert_eq!(bare_native.0, named_columns, "{bare_sql}: columns");
// `topk` selected, which this route states in its own declared column. The two routes name the
// one value column differently - the existing planner keeps the tree's `val`, the native route
// names it after the expression - so the columns are the existing answer's, with that name and
// the name column.
let mut expected_columns = bare_legacy
.0
.iter()
.filter(|column| *column != "val")
.cloned()
.collect::<Vec<_>>();
expected_columns.push("__name__".to_string());
expected_columns.push(expression_name("topk(3, native_dyn_topk)"));
expected_columns.sort();
assert_eq!(bare_native.0, expected_columns, "{bare_sql}: columns");
assert_eq!(
bare_native.1,
named_rows(bare_legacy.1, "native_dyn_topk"),
+38 -16
View File
@@ -1881,19 +1881,17 @@ fn result_columns(schema: &SchemaRef) -> Vec<(String, DataType, bool)> {
///
/// The tags are stated by the caller rather than read off the result: a vector that matches no
/// series reports no row to read them from, and still answers with its tag columns.
fn assert_native_result_schema(schema: &SchemaRef, tags: &[&str], context: &str) {
// The time index and the one field the tree reports, then one `Utf8` column per tag.
fn assert_native_result_schema(schema: &SchemaRef, tags: &[&str], value_name: &str, context: &str) {
// The time index and the one field the tree reports, then one `Utf8` column per tag. The
// native route names the value column after the expression, so the caller states that name;
// `value`/`val` are the legacy route's names.
let mut columns = vec![
(
if tags.is_empty() { "time" } else { "ts" }.to_string(),
DataType::Timestamp(TimeUnit::Millisecond, None),
false,
),
(
if tags.is_empty() { "value" } else { "val" }.to_string(),
DataType::Float64,
true,
),
(value_name.to_string(), DataType::Float64, true),
];
for name in tags {
columns.push(((*name).to_string(), DataType::Utf8, true));
@@ -2218,9 +2216,13 @@ async fn assert_raw_scan_fused_formats(observer: &NativeFusedObserver, frontend:
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
// The native route names its value column after the expression, so the
// type is what identifies it; `val` is only the legacy route's name.
let values = batch
.column_by_name("val")
.unwrap()
.columns()
.iter()
.find(|column| column.data_type() == &DataType::Float64)
.expect("one Float64 value column")
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
@@ -2744,7 +2746,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
};
let (plan, schema, rows) = promql_output(frontend, query, true, start, end).await;
assert_rows_eq(&rows, &expected, query);
assert_native_result_schema(&schema, tags, query);
assert_native_result_schema(&schema, tags, query, query);
let fused = assert_native_plan(&plan, scan_partitions, query);
assert_eq!(
fused["metrics"]["native.root.output_values"].as_u64(),
@@ -2769,13 +2771,33 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
);
assert_rows_eq(&legacy_rows, &expected, query);
assert_legacy_result_schema(&legacy_schema, tags, query);
// Both constant cases preserve the legacy time/value schema; computed vector columns are
// compared by role.
// Both routes report the time column and one Float64 value. The native route names that
// value after the expression, so the constant cases agree by role, not by header name.
if scan_partitions.is_none() {
assert_eq!(
result_columns(&legacy_schema),
result_columns(&schema),
"`{query}`: the two routes must report the same constant schema"
2,
schema.fields().len(),
"`{query}`: native constant schema"
);
assert_eq!(
2,
legacy_schema.fields().len(),
"`{query}`: legacy constant schema"
);
assert_eq!(
schema.field(0).data_type(),
legacy_schema.field(0).data_type(),
"`{query}`: time column type"
);
assert_eq!(
&DataType::Float64,
schema.field(1).data_type(),
"`{query}`: native value type"
);
assert_eq!(
&DataType::Float64,
legacy_schema.field(1).data_type(),
"`{query}`: legacy value type"
);
}
}
@@ -2788,7 +2810,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
observer.clear();
let (plan, schema, rows) = promql_output(frontend, query, true, "0", "2").await;
assert!(rows.is_empty(), "`{query}` reported {} rows", rows.len());
assert_native_result_schema(&schema, &["host"], query);
assert_native_result_schema(&schema, &["host"], query, query);
assert!(
executes_fused_series(&plan),
"`{query}` did not execute the fused route"