diff --git a/src/promql/src/extension_plan/fused_series.rs b/src/promql/src/extension_plan/fused_series.rs index 5226f5d0469..9dce9565c4b 100644 --- a/src/promql/src/extension_plan/fused_series.rs +++ b/src/promql/src/extension_plan/fused_series.rs @@ -377,8 +377,8 @@ struct FusedSeriesPayload { impl FusedSeries { /// Serializes the native tree and the binding required to reconstruct this plan node. - /// The PromQL text is included for diagnostics only and never drives execution; children - /// remain ordinary Substrait inputs. + /// The PromQL text identifies expression-named output fields; children remain ordinary + /// Substrait inputs. /// /// This is only the node's own encoding. Whether a planner is allowed to place this node on /// another machine is a separate decision, made by the planner and the transport, not here. @@ -497,6 +497,7 @@ impl FusedSeries { &node.tree, &node.output_tags()?, &node.binding.time_index, + &node.promql, )?; } Ok(node) @@ -1051,6 +1052,22 @@ impl FusedSeries { /// from the whole tree - a match narrows them, `or` joins both sides' and a wrapper around /// either keeps them - and the output schema is where it records that. fn output_tags(&self) -> DataFusionResult> { + if matches!(self.tree.root().output, NativeType::Scalar(_)) { + return Ok(Vec::new()); + } + if self.format == FusedOutputFormat::PlainRows { + // Roles are types here too: a value is a float or a native histogram, so every other + // non-time column of a row table is one of this output's labels, whatever its name. + return Ok(self + .output_schema + .fields() + .iter() + .filter(|field| { + field.name() != &self.binding.time_index && field.data_type() == &DataType::Utf8 + }) + .map(|field| field.name().clone()) + .collect()); + } let field_names = self .tree .root() @@ -1732,6 +1749,7 @@ fn validate_plain_output_schema( tree: &NativeTree, tags: &[String], time: &str, + promql: &str, ) -> DataFusionResult<()> { let fields = match &tree.root().output { NativeType::Samples(s) => s.fields.to_vec(), @@ -1755,7 +1773,7 @@ fn validate_plain_output_schema( )); } }; - classify_output(schema, &fields, tags, time)?; + classify_output(schema, &fields, tags, time, promql)?; Ok(()) } @@ -1844,13 +1862,73 @@ fn scalar_root_fields( }]) } +/// Whether a column of this type carries a field's samples. +/// +/// A role is a type: a field holds floats or the canonical native histogram struct, a tag is +/// `Utf8` and time is a timestamp. A name says nothing about which of those a column is. +fn is_value_column(data_type: &DataType) -> bool { + data_type == &DataType::Float64 || data_type == &native_histogram_value_type().as_arrow_type() +} + +/// Whether `names` name exactly the columns in `value_names`, one name per column. +fn names_value_columns(value_names: &[&str], names: &[String]) -> bool { + if names.len() != value_names.len() { + return false; + } + let mut declared = names.iter().map(String::as_str).collect::>(); + declared.sort_unstable(); + let mut carried = value_names.to_vec(); + carried.sort_unstable(); + declared == carried +} + +/// Names the fields in a row schema, in `series_fields` order. +/// +/// A caller names a value column either after the expression that produced it or after the tree's +/// field, so the typed value columns are what decide which: the expression names are used only +/// when they name exactly those columns, and the field names when those do. Deciding by type keeps +/// a tag that shares a field's name from being read as that field. The result stays in +/// `series_fields` order because a name is bound back to its field by that index, so a reordered +/// output still reads every column from its own field. +fn output_field_names( + output_schema: &SchemaRef, + series_fields: &[FieldSpec], + promql: &str, +) -> Vec { + let value_names = output_schema + .fields() + .iter() + .filter(|field| is_value_column(field.data_type())) + .map(|field| field.name().as_str()) + .collect::>(); + let expression_names = series_fields + .iter() + .map(|field| { + if series_fields.len() == 1 { + promql.to_string() + } else { + format!("{promql}.{}", field.name) + } + }) + .collect::>(); + if names_value_columns(&value_names, &expression_names) { + return expression_names; + } + series_fields + .iter() + .map(|field| field.name.to_string()) + .collect() +} + /// Classifies every output column, rejecting anything the conversion cannot produce. pub(crate) fn classify_output( output_schema: &SchemaRef, series_fields: &[FieldSpec], tags: &[String], time_column: &str, + promql: &str, ) -> DataFusionResult> { + let field_names = output_field_names(output_schema, series_fields, promql); if output_schema .fields() .iter() @@ -1873,11 +1951,6 @@ pub(crate) fn classify_output( "tag column {tag:?} of FusedSeriesExec cannot be the time column" ))); } - if series_fields.iter().any(|field| field.name.as_ref() == tag) { - return Err(DataFusionError::Plan(format!( - "tag column {tag:?} of FusedSeriesExec clashes with a series field" - ))); - } } let mut time_seen = false; @@ -1909,10 +1982,7 @@ pub(crate) fn classify_output( classification.push(OutputColumn::Time); continue; } - if let Some(index) = series_fields - .iter() - .position(|field| field.name.as_ref() == name) - { + if let Some(index) = field_names.iter().position(|field| field == name) { if std::mem::replace(&mut field_seen[index], true) { return Err(DataFusionError::Plan(format!( "duplicate field column {name:?} in FusedSeriesExec output schema" @@ -1977,7 +2047,7 @@ pub(crate) fn classify_output( if !seen { return Err(DataFusionError::Plan(format!( "missing field column {:?} in FusedSeriesExec output schema", - series_fields[index].name + field_names[index] ))); } } @@ -2485,7 +2555,7 @@ impl FusedSeriesExec { Vec::new() } else if scalar_root { let scalar_fields = scalar_root_fields(&output_schema, &tags, &time_column)?; - classify_output(&output_schema, &scalar_fields, &tags, &time_column)? + classify_output(&output_schema, &scalar_fields, &tags, &time_column, &promql)? } else { classify_output( &output_schema, @@ -2496,6 +2566,7 @@ impl FusedSeriesExec { .fields, &tags, &time_column, + &promql, )? }; let partition_count = inputs.first().map_or(1, |(input, _, _)| { diff --git a/src/promql/src/extension_plan/fused_series/tests.rs b/src/promql/src/extension_plan/fused_series/tests.rs index 73405bfd0f6..9820e6ea776 100644 --- a/src/promql/src/extension_plan/fused_series/tests.rs +++ b/src/promql/src/extension_plan/fused_series/tests.rs @@ -2599,6 +2599,149 @@ fn assert_decode_error(bytes: &[u8], expected: &str) { ); } +#[test] +fn renamed_reordered_value_columns_keep_field_identity_after_decode() { + let tree = rate_tree_of(two_field_schema()); + let schema = Arc::new(Schema::new(vec![ + Field::new(format!("{PROMQL}.{OTHER}"), DataType::Float64, true), + Field::new(HOST, DataType::Utf8, true), + Field::new( + TIME, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new(format!("{PROMQL}.{VALUE}"), DataType::Float64, true), + ])); + let node = FusedSeries::try_new( + Arc::from(PROMQL), + tree.clone(), + binding_of(&tree, two_field_schema(), vec![HOST.to_string()]), + vec![input_plan(frame_schema_of(&two_field_schema()))], + Arc::new(datafusion::common::DFSchema::try_from(schema.as_ref().clone()).unwrap()), + ) + .unwrap(); + let decoded = FusedSeries::deserialize(&node.serialize().unwrap()).unwrap(); + assert_eq!(node.schema(), decoded.schema()); + let tags = decoded.output_tags().unwrap(); + assert_eq!(vec![HOST], tags); + let classification = + super::classify_output(&schema, &two_field_schema().fields, &tags, TIME, PROMQL).unwrap(); + let chunk = EvaluatedChunk::try_new( + Series { + metric: SeriesLabels::try_new(None, Vec::new()).unwrap(), + time: TimeAxis::from_grid_spans(grid(), vec![GridSpan::new(0, 1)]).unwrap(), + fields: SeriesFields::try_from_vec(vec![ + Samples::from_float(Float64Array::from(vec![10.0])), + Samples::from_float(Float64Array::from(vec![20.0])), + ]) + .unwrap(), + }, + true, + ) + .unwrap(); + let batch = series_to_record_batch(&classification, &tags, &schema, &chunk).unwrap(); + for (name, expected) in [(OTHER, 20.0), (VALUE, 10.0)] { + let values = batch + .column_by_name(&format!("{PROMQL}.{name}")) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(expected, values.value(0)); + } +} + +/// A tag that carries the expression's name is a tag, not the statement's value. +/// +/// This is an old-style row schema: the statement is `host`, the schema names its one value +/// `field_0` and also carries the `host` tag. Deciding the value by whether a name occurs +/// anywhere in the schema would pick the tag as the value; the typed columns put the value at +/// `field_0` and the tag at `host`. +#[test] +fn a_tag_named_after_the_expression_is_not_the_value() { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("field_0", DataType::Float64, true), + Field::new(HOST, DataType::Utf8, true), + ])); + let fields = [FieldSpec { + name: "field_0".into(), + }]; + let classification = + super::classify_output(&schema, &fields, &[HOST.to_string()], TIME, HOST).unwrap(); + assert!( + matches!( + classification[1], + OutputColumn::Field { + index: 0, + kind: FieldKind::Float + } + ), + "{classification:?}" + ); + assert!( + matches!(classification[2], OutputColumn::Tag(0)), + "{classification:?}" + ); +} + +/// A reordered output still binds every value column to its own field. +/// +/// The expression names are matched as a set against the typed value columns, and the result is +/// kept in `series_fields` order - here `other` before `value` - because that order is what the +/// classification indexes the chunk's fields by. Names taken in output order would bind the first +/// column to `value`. +#[test] +fn reordered_expression_names_bind_fields_in_series_order() { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new(format!("{PROMQL}.{OTHER}"), DataType::Float64, true), + Field::new(HOST, DataType::Utf8, true), + Field::new(format!("{PROMQL}.{VALUE}"), DataType::Float64, true), + ])); + let series_schema = two_field_schema(); + let classification = super::classify_output( + &schema, + &series_schema.fields, + &[HOST.to_string()], + TIME, + PROMQL, + ) + .unwrap(); + assert!( + matches!( + classification[1], + OutputColumn::Field { + index: 1, + kind: FieldKind::Float + } + ), + "{classification:?}" + ); + assert!( + matches!(classification[2], OutputColumn::Tag(0)), + "{classification:?}" + ); + assert!( + matches!( + classification[3], + OutputColumn::Field { + index: 0, + kind: FieldKind::Float + } + ), + "{classification:?}" + ); +} + #[test] fn plain_rows_decode_rejects_invalid_schemas() { let node = fused(rate_tree()); @@ -2649,16 +2792,62 @@ fn plain_rows_decode_rejects_invalid_schemas() { false, ), Field::new(VALUE, DataType::Float64, true), + // A column's role is its type: a float here is not a tag, so this payload names a + // column the conversion cannot produce. Field::new(HOST, DataType::Float64, true), Field::new(NAME, DataType::Utf8, true), ]), - "must be Utf8", + "unknown column", ), ]; for (schema, expected) in malformed_schemas { assert_decode_error(&mutate_schema(&bytes, schema), expected); } + let renamed = mutate_envelope_payload(&bytes, |payload| { + let mut fields = payload.output_schema.fields().to_vec(); + let index = fields + .iter() + .position(|field| field.name() == VALUE) + .unwrap(); + fields[index] = Arc::new(Field::new(PROMQL, DataType::Float64, true)); + payload.output_schema = Schema::new(fields); + }); + assert!(FusedSeries::deserialize(&renamed).is_ok()); + // A value column is identified by type, so this payload types its one float column as neither + // a float nor a native histogram and names no value column the tree can read; a float value + // column that cannot be null is refused for its nullability, because a grid point may be + // missing. + for (data_type, nullable, expected) in [ + (DataType::Int64, true, "unknown column"), + (DataType::Float64, false, "must be nullable"), + ] { + let malformed = mutate_envelope_payload(&renamed, |payload| { + let mut fields = payload.output_schema.fields().to_vec(); + let index = fields + .iter() + .position(|field| field.name() == PROMQL) + .unwrap(); + fields[index] = Arc::new(Field::new(PROMQL, data_type, nullable)); + payload.output_schema = Schema::new(fields); + }); + assert_decode_error(&malformed, expected); + } + + // A scalar's value column is its one non-time column, so the wrong type is reported as a kind + // the conversion cannot produce. + let scalar_wrong_kind = mutate_envelope_payload(&scalar_bytes, |payload| { + let mut fields = payload.output_schema.fields().to_vec(); + let index = fields + .iter() + .position(|field| field.name() != TIME) + .expect("a scalar carries one value column"); + let name = fields[index].name().clone(); + fields[index] = Arc::new(Field::new(name, DataType::Int64, true)); + payload.output_schema = Schema::new(fields); + }); + assert_decode_error(&scalar_wrong_kind, "must be Float64"); + let scalar_decoded = FusedSeries::deserialize(&scalar_bytes).unwrap(); assert_eq!(scalar_decoded.schema(), scalar.schema()); diff --git a/src/promql/src/native/segment.rs b/src/promql/src/native/segment.rs index ca7eb7994b6..f4f9f3ffa1d 100644 --- a/src/promql/src/native/segment.rs +++ b/src/promql/src/native/segment.rs @@ -21,8 +21,8 @@ //! nodes, recursing into a cut that wraps a whole-vector consumer. //! //! Everything here reads the bound tree alone. The columns a node publishes are the planner's -//! answer, so the builders ask for them through [`OutputTags`] and [`RootSchema`] instead of -//! deriving them a second time. +//! answer, so the builders ask for them through [`OutputTags`], [`FieldNames`] and [`RootSchema`] +//! instead of deriving them a second time. use std::sync::Arc; @@ -56,11 +56,20 @@ use crate::vector_match::Cardinality; /// rather than deriving the columns a second time. pub type OutputTags = fn(&NativeNode, &[Vec]) -> Option>; -/// The row schema a statement root publishes, or `None` when its columns cannot be named once. +/// Derives the value column names a statement's expression gives the compiled `fields`. +/// +/// Like [`OutputTags`], this is the planner's answer: a segment publishes the values its tree +/// computes under the names the statement gave them, so the builders ask through this leaf instead +/// of publishing the compiler's own field names. The tree alone does not carry the statement's +/// expression, which is why the expression text is an input rather than something derived here. +pub type FieldNames = fn(&str, &[String]) -> Vec; + +/// The row schema a statement root publishes for `promql`, or `None` when its columns cannot be +/// named once. /// /// Like [`OutputTags`], this is the caller's answer: a segment that publishes rows publishes -/// exactly the columns the planner would have published for the same tree. -pub type RootSchema = fn(&NativeTree, &[Vec], &str, FieldKind) -> Option; +/// exactly the columns the planner would have published for the same statement. +pub type RootSchema = fn(&str, &NativeTree, &[Vec], &str, FieldKind) -> Option; /// Whether `node` reads samples point-wise, so that one partitioned execution is one answer. /// @@ -496,8 +505,8 @@ pub const NATIVE_ORDERING: SeriesOrdering = SeriesOrdering::Contiguous; /// The columns a [`FusedSeries`] node produces. /// -/// With `experimental_native_promql` enabled, a value column is named after its compiled root -/// field (for example, `field_0`), not legacy planner expression text. Legacy headers in +/// With `experimental_native_promql` enabled, a value column is named by the caller's [`FieldNames`] +/// answer for the statement's expression, not legacy planner expression text. Legacy headers in /// `tests/cases/**` are unaffected because the native route is opt-in and off by default. /// /// The shape is the one the existing unary pipeline consumes — the time index, then the series @@ -623,6 +632,7 @@ pub fn canonical_boundary_stages( format: FusedOutputFormat, root_kind: FieldKind, output_tags_of: OutputTags, + field_names: FieldNames, ) -> Option<(LogicalPlan, Vec)> { let NativeExpr::Merge { input } = &tree.root().expr else { return None; @@ -644,7 +654,12 @@ pub fn canonical_boundary_stages( // declares it exactly as the other rows layouts do. The tags stay the walk's own: they name the // labels the frames carry, and the binding below is built from them. let row_tags = with_row_name_column(tree.root(), output_tags.clone()); - let output_schema = fused_output_schema(&fields, &row_tags, time_index, root_kind).ok()?; + // The values are named after the expression, as in every other rows layout of this statement. + // A schema this boundary cannot declare declines here: the caller's error type is not one this + // crate can build, so a name collision is reported by declining rather than as an internal + // error, and the caller keeps the plan it would have chosen. + let value_names = field_names(&promql, &fields); + let output_schema = fused_output_schema(&value_names, &row_tags, time_index, root_kind).ok()?; // The producer: the statement's own tree, run per partition, exporting a frame. It runs above // the regions, where its router collects each label set from every region's contribution. @@ -766,6 +781,7 @@ fn build_evaluated_segment( time_index: &str, root_kind: FieldKind, output_tags_of: OutputTags, + field_names: FieldNames, ) -> Option<(LogicalPlan, Vec)> { let mut sources = Vec::new(); collect_segment_inputs(node, &mut sources); @@ -829,6 +845,7 @@ fn build_evaluated_segment( FusedOutputFormat::SeriesFrame, root_kind, output_tags_of, + field_names, )?; return Some((plan, tags)); } @@ -959,7 +976,8 @@ fn build_inner_segment( /// per-series prefixes do not need that gather, so each becomes a segment of its own and keeps the /// partitions it was scanned with. Global consumers and canonical merges are cut in their original /// order, and recursive segments publish frames for their parents to consume. Only the statement -/// root publishes rows. +/// root publishes rows, and it does so under the statement's own value names: [`FieldNames`] +/// answers those, and the rows schema asks [`RootSchema`] for the rest of the layout. #[allow(clippy::too_many_arguments, clippy::type_complexity)] pub fn segment_tree( promql: Arc, @@ -977,6 +995,7 @@ pub fn segment_tree( root_kind: FieldKind, remote_raw_boundary: bool, output_tags_of: OutputTags, + field_names: FieldNames, root_schema: RootSchema, ) -> Option { // Only a tree the analyzer gathers is worth splitting: its per-series prefixes then run where @@ -1061,6 +1080,7 @@ pub fn segment_tree( root_kind, remote_raw_boundary, output_tags_of, + field_names, root_schema, ); match plan { @@ -1096,6 +1116,7 @@ pub fn segment_tree( time_index, root_kind, output_tags_of, + field_names, ) }); let (plan, cut_tags) = built?; @@ -1164,6 +1185,7 @@ pub fn segment_tree( format, root_kind, output_tags_of, + field_names, ) .map(|(plan, _)| plan); } @@ -1176,7 +1198,9 @@ pub fn segment_tree( return None; } let output_schema = match format { - FusedOutputFormat::PlainRows => root_schema(&parent, &child_tags, time_index, root_kind)?, + FusedOutputFormat::PlainRows => { + root_schema(&promql, &parent, &child_tags, time_index, root_kind)? + } FusedOutputFormat::SeriesFrame => { let Ok(frame) = FusedSeries::series_frame_schema(&parent) else { return None; diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs index 9d910e9782b..af68a9d70c9 100644 --- a/src/query/src/planner.rs +++ b/src/query/src/planner.rs @@ -1866,9 +1866,8 @@ mod tests { ); } - /// A statement that reads no table plans natively, and there is no region to place anything in: - /// the frontend evaluates the fused node under the native column names rather than the ones the - /// existing planner derives from an expression. + /// A statement that reads no table plans natively over the wire, where there is no region to + /// place anything in: the frontend evaluates the fused node with expression-named values. #[tokio::test] async fn test_remote_native_promql_plans_a_data_free_statement_natively() { let engine = create_promql_test_engine_with( @@ -1879,35 +1878,26 @@ mod tests { false, ); let query_ctx = native_discovery_query_ctx(); - for (query, expected_columns, expected_values, expected_labels) in [ - // `time()` answers the evaluation grid in seconds. - ( - "time()", - &["time", "value"][..], - &[0.0, 5.0, 10.0][..], - &[][..], - ), + for (query, expected_values, expected_labels, name_column) in [ + // `time()` answers the evaluation grid in seconds. It is a scalar root: it publishes + // one value and no series, so its rows layout declares no name column. + ("time()", &[0.0, 5.0, 10.0][..], &[][..], false), // `vector(1)` is a samples root: its rows layout declares the per-row name column, // which names no row because the vector carries no series identity. - ( - "vector(1)", - &["time", "value", "__name__"][..], - &[1.0, 1.0, 1.0][..], - &[][..], - ), + ("vector(1)", &[1.0, 1.0, 1.0][..], &[][..], true), ( r#"label_replace(vector(1), "host", "host1", "", "")"#, - &["time", "value", "host", "__name__"][..], &[1.0, 1.0, 1.0][..], &[("host", "host1")][..], + true, ), // A synthesized `__tsid` is published under the statement's own columns: the stripping // projection above the node is read through rather than mistaken for a remote plan. ( r#"label_replace(vector(1), "__tsid", "x", "", "")"#, - &["time", "value", "__name__"][..], &[1.0, 1.0, 1.0][..], &[][..], + true, ), ] { let plan = plan_remote_native_promql(&engine, &query_ctx, query).await; @@ -1917,7 +1907,15 @@ mod tests { !display.contains("greptime_value"), "{query} keeps the native value column: {display}" ); - // A data-free statement has no input at all, so nothing is scanned or left empty. + // A data-free statement has no input at all, so nothing is scanned or left empty. The + // value is named after the expression, and a samples root's rows keep the name column + // last. + let expression = promql_parser::parser::parse(query).unwrap().to_string(); + let mut expected_columns = vec!["time", expression.as_str()]; + expected_columns.extend(expected_labels.iter().map(|(name, _)| *name)); + if name_column { + expected_columns.push("__name__"); + } assert_eq!( expected_columns, plan.schema() @@ -1931,7 +1929,7 @@ mod tests { assert_regionless_physical_plan(&physical, query); assert_eq!( expected_values, - numeric_values(&batches, "value").as_slice(), + numeric_values(&batches, &expression).as_slice(), "{query}: {batches:?}" ); for (label, value) in expected_labels { diff --git a/src/query/src/promql/error.rs b/src/query/src/promql/error.rs index 78e7289b479..dd23e6730b9 100644 --- a/src/query/src/promql/error.rs +++ b/src/query/src/promql/error.rs @@ -58,6 +58,15 @@ pub enum Error { location: Location, }, + #[snafu(display( + "duplicate native PromQL output column name(s): {names}; time index, value and tag columns must have distinct names" + ))] + DuplicateOutputColumns { + names: String, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Unexpected plan or expression: {}", desc))] UnexpectedPlanExpr { desc: String, @@ -281,7 +290,8 @@ impl ErrorExt for Error { | SystemTimeOutOfRange { .. } | AtModifierTimestampOutOfRange { .. } | InvalidRegularExpression { .. } - | InvalidDestinationLabelName { .. } => StatusCode::InvalidArguments, + | InvalidDestinationLabelName { .. } + | DuplicateOutputColumns { .. } => StatusCode::InvalidArguments, UnknownTable { .. } => StatusCode::Internal, diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index c267c729ba0..7490db01fa6 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -62,7 +62,7 @@ use datafusion_expr::expr_fn::when; use datafusion_expr::utils::{conjunction, disjunction}; use datafusion_expr::{ExprSchemable, Literal, SortExpr, TableSource, ident, lit}; use datafusion_functions::core::coalesce; -use datatypes::arrow::datatypes::{DataType as ArrowDataType, TimeUnit as ArrowTimeUnit}; +use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit as ArrowTimeUnit}; use datatypes::data_type::{ConcreteDataType, DataType as GreptimeDataType}; use itertools::Itertools; use once_cell::sync::Lazy; @@ -1435,13 +1435,11 @@ impl PromPlanner { let right_ctx = self.ctx.clone(); (left, left_ctx, right, right_ctx) }; - let left_field_columns = left_context.field_columns.clone(); let left_time_index_column = left_context.time_index_column.clone(); self.ctx = left_context.clone(); let mut left_table_ref = self .table_ref() .unwrap_or_else(|_| TableReference::bare("")); - let right_field_columns = right_context.field_columns.clone(); let right_time_index_column = right_context.time_index_column.clone(); self.ctx = right_context.clone(); let mut right_table_ref = self @@ -1450,7 +1448,8 @@ impl PromPlanner { // Both operands are planned first because only the planned contexts tell a tag // from a value field. The rewrite merely adds matchers to a selector, so the - // table reference, time index and field columns captured above stay valid. + // table reference and time index captured above stay valid. The field columns are + // read after this rewrite, because an operand it re-plans brings its own. if let Some(rewritten) = matching_filters::propagate( binary_expr, &left_context.tag_columns, @@ -1481,6 +1480,8 @@ impl PromPlanner { self.ctx = right_context.clone(); } + let left_field_columns = left_context.field_columns.clone(); + let right_field_columns = right_context.field_columns.clone(); let left_is_empty_metric = Self::is_empty_metric(&left_input); let right_is_empty_metric = Self::is_empty_metric(&right_input); @@ -4019,12 +4020,28 @@ impl PromPlanner { query_engine_state, )?; - // Reserve the current field columns except the `dst_label`. - for value in &self.ctx.field_columns { - if *value != dst_label { - let expr = DfExpr::Column(Column::from_name(value)); - exprs.push(expr); + // A destination label is a tag, and a tag is published under its own name. A + // value field of that name would have to be dropped or renamed to make room, and + // neither is what the statement says, so the collision is the statement's error + // rather than a value quietly going missing. + if self + .ctx + .field_columns + .iter() + .any(|field| field == &dst_label) + { + return UnsupportedExprSnafu { + name: format!( + "label_join destination label {dst_label:?} collides with a value field" + ), } + .fail(); + } + + // Reserve the current field columns. + for value in &self.ctx.field_columns { + let expr = DfExpr::Column(Column::from_name(value)); + exprs.push(expr); } // Remove it from tag columns if exists to avoid duplicated column names @@ -4040,12 +4057,10 @@ impl PromPlanner { if let Some((replace_expr, dst_label)) = self.build_regexp_replace_label_expr(&mut other_input_exprs, input_schema)? { - // Reserve the current field columns except the `dst_label`. + // Reserve the current field columns. for value in &self.ctx.field_columns { - if *value != dst_label { - let expr = DfExpr::Column(Column::from_name(value)); - exprs.push(expr); - } + let expr = DfExpr::Column(Column::from_name(value)); + exprs.push(expr); } // Remove it from tag columns if exists to avoid duplicated column names @@ -4375,6 +4390,22 @@ impl PromPlanner { // Validate the destination label name Self::validate_label_name(&dst_label)?; + // A destination label is a tag, and a tag is published under its own name. A value field + // of that name would have to be dropped or renamed to make room, so report the collision + // before any no-op path below can silently decline the rewrite. + if self + .ctx + .field_columns + .iter() + .any(|field| field == &dst_label) + { + return UnsupportedExprSnafu { + name: format!( + "label_replace destination label {dst_label:?} collides with a value field" + ), + } + .fail(); + } let replacement = match other_input_exprs.pop_front() { Some(DfExpr::Literal(ScalarValue::Utf8(Some(r)), _)) => r, other => UnexpectedPlanExprSnafu { @@ -6668,11 +6699,18 @@ impl PromPlanner { fn strip_tsid_column(&self, plan: LogicalPlan) -> Result { let schema = plan.schema(); - if !schema - .fields() - .iter() - .any(|field| field.name() == DATA_SCHEMA_TSID_COLUMN_NAME) - { + // A series identifier is the internal `UInt64` key or the `Utf8` label synthesized from + // it, and only those two representations are stripped. The same name on any other column + // is the statement's own answer - `count(m)` aliased to `__tsid` answers an `Int64`, and a + // value is `Float64` or the canonical native histogram struct - so a name is not a role. + fn is_internal_tsid(field: &Field) -> bool { + field.name() == DATA_SCHEMA_TSID_COLUMN_NAME + && matches!( + field.data_type(), + ArrowDataType::UInt64 | ArrowDataType::Utf8 + ) + } + if !schema.fields().iter().any(|field| is_internal_tsid(field)) { return Ok(plan); } @@ -6680,7 +6718,7 @@ impl PromPlanner { // the columns by their original qualified names. let project_exprs = schema .iter() - .filter(|(_, field)| field.name() != DATA_SCHEMA_TSID_COLUMN_NAME) + .filter(|(_, field)| !is_internal_tsid(field)) .map(|(qualifier, field)| { DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone())) }) diff --git a/src/query/src/promql/planner/native.rs b/src/query/src/promql/planner/native.rs index da6ff6c4a79..acb7dbba2f3 100644 --- a/src/query/src/promql/planner/native.rs +++ b/src/query/src/promql/planner/native.rs @@ -77,11 +77,11 @@ use promql_parser::parser::{ EvalStmt, Expr as PromExpr, MatrixSelector, ParenExpr, SubqueryExpr, VectorSelector, }; use session::context::{MetricDiscoveryScope, QueryContextRef}; -use snafu::ResultExt; +use snafu::{ResultExt, ensure}; use store_api::metric_engine_consts::{DATA_SCHEMA_TABLE_ID_COLUMN_NAME, METRIC_ENGINE_NAME}; use store_api::region_engine::RawLabelSource; -use crate::promql::error::{DataFusionPlanningSnafu, Result}; +use crate::promql::error::{DataFusionPlanningSnafu, DuplicateOutputColumnsSnafu, Result}; use crate::promql::native_compile::{ InfoInput, bind_evaluated_binary, compile_selectors_with_grid, compile_statement_with_grid, info_input, statement_grid, subquery_child_grid, @@ -528,6 +528,8 @@ impl PromPlanner { { return Ok(None); } + // The expression names the value columns, and it is the label the node reports. + let expression = expr.to_string(); // What a consumer of this composition reads, and the layout the node publishes. The row // layout names one column per value; the frame layout is the evaluated frame this tree's // own root declares, which is what a frame consumer decodes. @@ -542,8 +544,13 @@ impl PromPlanner { }; let output_schema = match format { FusedOutputFormat::PlainRows => { - let Some(schema) = - Self::fused_root_schema(&tree, &source_tags, &left_time, root_kind) + let Some(schema) = Self::fused_root_schema( + &expression, + &tree, + &source_tags, + &left_time, + root_kind, + )? else { return Ok(None); }; @@ -562,7 +569,7 @@ impl PromPlanner { } }; let Ok(node) = FusedSeries::try_new_with_format( - Arc::from(expr.to_string()), + Arc::from(expression.as_str()), tree, FusedSeriesBinding { grid: Arc::clone(&left_grid), @@ -586,8 +593,8 @@ impl PromPlanner { /// Plans `prom_expr` as one native node, or returns `None` to keep the existing plan. /// - /// The compiler's bound tree and selector schemas determine the native output names; the existing - /// planner is not consulted for capability or schema approval. + /// The expression names the value columns; the compiler's bound tree and selector schemas + /// determine their values. The existing planner is not consulted for capability or schema approval. /// /// The route is offered a whole expression because the compiler folds a range function and the math /// functions over it into one tree: offering a sub-expression would compile a tree that computes @@ -606,6 +613,10 @@ impl PromPlanner { if !self.native_fused { return Ok(None); } + // The expression names the value columns of every rows layout this attempt publishes, and the + // node's own label follows the same text. It is the whole statement's expression even when a + // subquery root compiles its inner one, because that is the statement the reader wrote. + let expression = prom_expr.to_string(); // A range selector returns raw samples, which have no evaluation grid to bind. A subquery root // instead returns its inner expression on the subquery's child grid, and a range query has no grid // to bind a matrix root to. @@ -689,7 +700,13 @@ impl PromPlanner { let Some(values) = native_root_fields(tree.root()) else { return Ok(None); }; - let time_index = if values + // A label named `time` is the one collision the default time column can have: the + // value columns are named after the expression, so they never take that name. The + // time column is this route's own synthetic name - no label or value is renamed - so + // a statement whose output already holds a `time` column publishes its grid under the + // first free `native_time_N` instead of dropping or renaming the label. + let value_names = expression_field_names(&expression, &values); + let time_index = if value_names .iter() .chain(&output_tags) .any(|name| name == DEFAULT_TIME_INDEX_COLUMN) @@ -697,7 +714,7 @@ impl PromPlanner { (0usize..) .map(|index| format!("native_time_{index}")) .find(|candidate| { - !values + !value_names .iter() .chain(&output_tags) .any(|name| name == candidate) @@ -717,6 +734,7 @@ impl PromPlanner { // A data-free tree reads no input, so every field it answers is a float. let output_schema = match format { FusedOutputFormat::PlainRows => fused_output_schema( + &expression, &values, &with_row_name_column(tree.root(), output_tags.clone()), &time_index, @@ -732,7 +750,7 @@ impl PromPlanner { } }; let node = FusedSeries::try_new_with_format( - Arc::from(prom_expr.to_string()), + Arc::from(expression.as_str()), tree, FusedSeriesBinding { grid: Arc::clone(&grid), @@ -1466,7 +1484,7 @@ impl PromPlanner { if selectors.len() == 1 && matches!(&tree.root().expr, NativeExpr::Merge { input } if pointwise_operand(input) && canonical_local_prefix(input) && !contains_global_consumer(input)) && let Some((plan, _)) = canonical_boundary_stages( - Arc::from(prom_expr.to_string()), + Arc::from(expression.as_str()), &tree, &children, &source_tags, @@ -1477,6 +1495,7 @@ impl PromPlanner { format, root_kind, fused_output_tags, + expression_field_names, ) { return Ok(Some(native_planned( @@ -1510,7 +1529,7 @@ impl PromPlanner { // caller asked for: it names one column per value the partial state accumulates, // while the encoded frame carries those values inside its own layout. let Some(value_schema) = - Self::fused_root_schema(&tree, &source_tags, &time, root_kind) + Self::fused_root_schema(&expression, &tree, &source_tags, &time, root_kind)? else { return Ok(None); }; @@ -1525,7 +1544,7 @@ impl PromPlanner { }; let publish_grid = Arc::clone(&grid); let node = plan_split_aggregate( - Arc::from(prom_expr.to_string()), + Arc::from(expression.as_str()), partial, final_tree, FusedSeriesBinding { @@ -1556,7 +1575,7 @@ impl PromPlanner { if required_placement || matches!(locality, InputLocality::SinglePartition) { let segment_metric = if cross_table { None } else { metric.clone() }; let plan = segment_tree( - Arc::from(prom_expr.to_string()), + Arc::from(expression.as_str()), &tree, &children, &source_tags, @@ -1575,7 +1594,8 @@ impl PromPlanner { // that is not a tag, and a partitioned table whose occurrences are source-backed. any_split_series || (per_source_inputs && any_partitioned), fused_output_tags, - PromPlanner::fused_root_schema, + expression_field_names, + PromPlanner::segmented_root_schema, ); if let Some(plan) = plan { return Ok(Some(native_planned( @@ -1609,20 +1629,9 @@ impl PromPlanner { let Some(root_fields) = native_root_fields(tree.root()) else { return Ok(None); }; - // A root field can share a name with a tag or the time index, and under `or` the right - // table's tags join the left table's columns. The output names each column once, so such a - // statement keeps the existing plan. - let mut column_names = std::collections::HashSet::new(); - if !std::iter::once(&time) - .chain(&root_fields) - .chain(&output_tags) - .all(|name| column_names.insert(name.as_str())) - { - return Ok(None); - } let output_schema = match format { FusedOutputFormat::PlainRows => { - fused_output_schema(&root_fields, &output_tags, &time, root_kind)? + fused_output_schema(&expression, &root_fields, &output_tags, &time, root_kind)? } FusedOutputFormat::SeriesFrame => { let frame = @@ -1645,7 +1654,7 @@ impl PromPlanner { // binding is that grid rather than a per-shape choice. let publish_grid = Arc::clone(&grid); let node = FusedSeries::try_new_with_format( - Arc::from(prom_expr.to_string()), + Arc::from(expression.as_str()), tree, FusedSeriesBinding { grid: publish_grid, @@ -1673,36 +1682,56 @@ impl PromPlanner { /// The columns a [`FusedSeries`] node over `tree` produces. /// - /// Field and tag names are derived from the native tree; a walk that cannot derive a tag set declines - /// rather than consulting the legacy planner. + /// Fields and tags are derived from the native tree; the expression names the output fields. + /// If this walk cannot derive a tag set, the native route declines rather than consulting the + /// legacy planner. /// - /// `None` when the root is not a value the node can publish, or when a field would share a name with a - /// tag or the time index. + /// `Ok(None)` when the root is not a value the node can publish. A schema the output cannot + /// declare is an error rather than a decline: duplicate names violate the expression-named + /// native output contract, so construction reports the collision instead of publishing a + /// layout the reader cannot address. fn fused_root_schema( + expression: &str, + tree: &NativeTree, + source_tags: &[Vec], + time: &str, + root_kind: FieldKind, + ) -> Result> { + let output_tags = match &tree.root().output { + NativeType::Scalar(_) => Vec::new(), + NativeType::Samples(_) => { + let Some(tags) = fused_output_tags(tree.root(), source_tags) else { + return Ok(None); + }; + with_row_name_column(tree.root(), tags) + } + NativeType::AggregateState(_) => return Ok(None), + }; + let Some(root_fields) = native_root_fields(tree.root()) else { + return Ok(None); + }; + // The kind is the one the statement's own root answers, derived before segmentation cut + // anything. It cannot be recomputed here: a cut prefix is replaced by an input leaf whose + // samples are already evaluated, so walking this tree would read every such leaf as the + // kind its scan carried. `count_over_time(h[5m]) + on (__name__) ...` answers floats, and + // its replaced leaf carries floats, while the scan below was a histogram. + fused_output_schema(expression, &root_fields, &output_tags, time, root_kind).map(Some) + } + + /// The rows layout a segmented statement root publishes, under the segment builders' callback. + /// + /// A segment cannot carry this crate's error type, so a layout it cannot declare is a decline + /// here: the route keeps the plan the existing planner would have built for the same statement. + fn segmented_root_schema( + expression: &str, tree: &NativeTree, source_tags: &[Vec], time: &str, root_kind: FieldKind, ) -> Option { - let output_tags = match &tree.root().output { - NativeType::Scalar(_) => Vec::new(), - NativeType::Samples(_) => { - with_row_name_column(tree.root(), fused_output_tags(tree.root(), source_tags)?) - } - NativeType::AggregateState(_) => return None, - }; - let root_fields = native_root_fields(tree.root())?; - let mut column_names = std::collections::HashSet::new(); - let columns = std::iter::once(time) - .chain(root_fields.iter().map(String::as_str)) - .chain(output_tags.iter().map(String::as_str)); - if !columns.clone().all(|name| column_names.insert(name)) { - return None; - } - // The kind is the one the statement's own root answers, derived before segmentation cut anything. - // It cannot be recomputed here: a cut prefix is replaced by an input leaf whose samples are - // already evaluated. - fused_output_schema(&root_fields, &output_tags, time, root_kind).ok() + Self::fused_root_schema(expression, tree, source_tags, time, root_kind) + .ok() + .flatten() } /// The statement being planned, rebuilt from the evaluation numbers the planner holds. @@ -1821,18 +1850,67 @@ fn native_root_fields(node: &NativeNode) -> Option> { } } -/// The output schema of a fused node: its time index, one column per field, then its tags. +/// The names an expression gives the value columns of `fields`. /// -/// The columns themselves are named by [`segment::fused_output_schema`], which is where a segment -/// asks for the same layout; this is that answer under this crate's error type, which is what the -/// planner's own call sites report. +/// One value is named after the whole expression; several append their compiled field names to it, +/// because one name cannot stand for more than one column. Only the published layout is named this +/// way: the fields inside the native tree keep the names the compiler gave them. +fn expression_field_names(expression: &str, fields: &[String]) -> Vec { + fields + .iter() + .map(|field| { + if fields.len() == 1 { + expression.to_string() + } else { + format!("{expression}.{field}") + } + }) + .collect() +} + +/// The columns a [`FusedSeries`] node produces: its time index, one column per value, then its tags. +/// +/// A single value is named after the expression, and multiple values append their compiled field +/// names to it, so a value is never published under a name the reader cannot address. The names +/// inside the native tree are unchanged. +/// +/// The layout itself is [`segment::fused_output_schema`]'s, which is where a segment asks for the +/// same columns; this is that answer under this crate's error type, which is what the planner's own +/// call sites report. fn fused_output_schema( + expression: &str, fields: &[String], tags: &[String], time_index: &str, field_kind: FieldKind, ) -> Result { - segment::fused_output_schema(fields, tags, time_index, field_kind) + let value_names = expression_field_names(expression, fields); + // A name appears twice when a value named after its expression - a bare selector is named after + // its metric - lands on the table's own time index or on one of its tags, or when two values + // take one name. `DFSchema` rejects that, but reports only that building the plan failed, which + // names no column; the collision is detected here so the error names the columns that collide. + let mut names = Vec::with_capacity(value_names.len() + tags.len() + 1); + names.push(time_index.to_string()); + names.extend(value_names.iter().cloned()); + names.extend(tags.iter().cloned()); + let mut occurrences: std::collections::HashMap<&str, usize> = + std::collections::HashMap::with_capacity(names.len()); + for name in &names { + *occurrences.entry(name.as_str()).or_default() += 1; + } + let mut reported = std::collections::HashSet::new(); + let duplicates = names + .iter() + .filter(|name| occurrences[name.as_str()] > 1 && reported.insert(name.as_str())) + .map(String::as_str) + .collect::>(); + ensure!( + duplicates.is_empty(), + DuplicateOutputColumnsSnafu { + names: duplicates.join(", ") + } + ); + segment::fused_output_schema(&value_names, tags, time_index, field_kind) .context(DataFusionPlanningSnafu) } @@ -2911,8 +2989,10 @@ mod tests { ("x or y", two()), ] { let tree = compiled_test_tree(query); - let schema = PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float) - .unwrap_or_else(|| panic!("{query}: a samples root declares a rows layout")); + let schema = + PromPlanner::fused_root_schema(query, &tree, &sources, "ts", FieldKind::Float) + .unwrap() + .unwrap_or_else(|| panic!("{query}: a samples root declares a rows layout")); let names = schema .fields() .iter() @@ -2932,9 +3012,13 @@ mod tests { assert_eq!(ArrowDataType::Utf8, *name.data_type(), "{query}"); } let scalar = compiled_test_tree("1 + 2"); - let schema = PromPlanner::fused_root_schema(&scalar, &[], "ts", FieldKind::Float).unwrap(); + let schema = PromPlanner::fused_root_schema("1 + 2", &scalar, &[], "ts", FieldKind::Float) + .unwrap() + .unwrap(); assert_eq!( - vec!["ts", "value"], + // A scalar root declares no name column, and its one value is named after the + // expression, as every other rows layout names its values. + vec!["ts", "1 + 2"], schema .fields() .iter() @@ -2948,10 +3032,11 @@ mod tests { fn a_present_name_tag_is_not_declared_twice() { let sources = [vec!["host".to_string(), METRIC_NAME.to_string()]]; let tree = compiled_test_tree("x"); - let schema = - PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float).unwrap(); + let schema = PromPlanner::fused_root_schema("x", &tree, &sources, "ts", FieldKind::Float) + .unwrap() + .unwrap(); assert_eq!( - vec!["ts", "v0", "host", METRIC_NAME], + vec!["ts", "x", "host", METRIC_NAME], schema .fields() .iter() @@ -2960,19 +3045,34 @@ mod tests { ); } - /// A value column that is itself named `__name__` is a column of the result, not the name of a - /// row: the layout names each column once, so such a statement keeps the existing plan. + /// The value columns are named after the expression, so a tree whose own field is named + /// `__name__` still publishes that value under the expression's name - the name column is the + /// layout's own. A time index of that name collides with it, and the collision is reported + /// rather than published as a layout the reader cannot address. #[test] - fn a_column_named_after_the_metric_name_declines_the_rows_layout() { + fn a_time_index_named_after_the_metric_name_is_reported() { let sources = [vec!["host".to_string()]]; + // The field is renamed by the expression, so it no longer stands where the name column is. let tree = compiled_test_tree_with_fields("x", &[METRIC_NAME]); - assert!(PromPlanner::fused_root_schema(&tree, &sources, "ts", FieldKind::Float).is_none()); - // A time index of that name collides the same way. - let tree = compiled_test_tree("x"); - assert!( - PromPlanner::fused_root_schema(&tree, &sources, METRIC_NAME, FieldKind::Float) - .is_none() + let schema = PromPlanner::fused_root_schema("x", &tree, &sources, "ts", FieldKind::Float) + .unwrap() + .unwrap_or_else(|| panic!("the value is named after the expression")); + assert_eq!( + vec!["ts", "x", "host", METRIC_NAME], + schema + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect::>() ); + // A time index of that name collides with the row-name column, and the error names it. + let tree = compiled_test_tree("x"); + let error = + PromPlanner::fused_root_schema("x", &tree, &sources, METRIC_NAME, FieldKind::Float) + .expect_err("a time index named after the metric collides with the name column"); + let message = error.to_string().to_lowercase(); + assert!(message.contains("duplicate"), "{message}"); + assert!(message.contains(METRIC_NAME), "{message}"); } /// A `__name__` rewrite publishes a name column, canonicalizes the complete runtime label set, @@ -3145,7 +3245,9 @@ mod tests { .map(|field| field.name.as_ref()) .collect::>() ); + let expression = "sum by (zone, host) ((x))"; let output = fused_output_schema( + expression, &["v0".to_string(), "v1".to_string()], &["zone".to_string(), "host".to_string()], "ts", @@ -3153,7 +3255,13 @@ mod tests { ) .unwrap(); assert_eq!( - vec!["ts", "v0", "v1", "zone", "host"], + vec![ + "ts".to_string(), + format!("{expression}.v0"), + format!("{expression}.v1"), + "zone".to_string(), + "host".to_string(), + ], output .fields() .iter() diff --git a/src/query/src/promql/planner/test.rs b/src/query/src/promql/planner/test.rs index 7f56f3fd2c3..659e16fb3d4 100644 --- a/src/query/src/promql/planner/test.rs +++ b/src/query/src/promql/planner/test.rs @@ -2375,11 +2375,16 @@ async fn native_metric_discovery_without_a_matching_metric_answers_with_no_rows( async fn native_metric_discovery_without_a_matching_metric_answers_absent_and_or() { // `absent` labels its one series from the selector's equality matchers, which is what the // existing planner's `Absent` node does with the same matchers, and the value is reported - // under the name the empty input's one field has. - let (plan, batches) = execute_no_match_discovery("absent({job=\"api\"})").await; + // under the expression's name. + let absent = "absent({job=\"api\"})"; + let (plan, batches) = execute_no_match_discovery(absent).await; assert!(find_fused_series(&plan).is_some(), "{plan:?}"); assert_eq!(1, batch_rows(&batches), "{plan:?}"); - assert_eq!(vec![1.0], numeric_values(&batches, "value"), "{plan:?}"); + assert_eq!( + vec![1.0], + numeric_values(&batches, &parser::parse(absent).unwrap().to_string()), + "{plan:?}" + ); assert_eq!( vec!["api".to_string()], batches @@ -2399,10 +2404,15 @@ async fn native_metric_discovery_without_a_matching_metric_answers_absent_and_or // `vector(1)` has no labels of its own, so the row it contributes holds none. This is the set // operator `or`, which the native route answers over an empty side. - let (plan, batches) = execute_no_match_discovery("{job=\"api\"} or vector(1)").await; + let or = "{job=\"api\"} or vector(1)"; + let (plan, batches) = execute_no_match_discovery(or).await; assert!(find_fused_series(&plan).is_some(), "{plan:?}"); assert_eq!(1, batch_rows(&batches), "{plan:?}"); - assert_eq!(vec![1.0], numeric_values(&batches, "value"), "{plan:?}"); + assert_eq!( + vec![1.0], + numeric_values(&batches, &parser::parse(or).unwrap().to_string()), + "{plan:?}" + ); } /// A zero-match discovery selector is a real input even when another selector reads a metric. @@ -2436,7 +2446,8 @@ async fn native_zero_match_discovery_with_real_selector_binds_both_inputs() { .iter() .map(|field| field.name().as_str()) .collect::>(); - for column in ["timestamp", "field_0", "tag_0", METRIC_NAME] { + let expression = parser::parse(query).unwrap().to_string(); + for column in ["timestamp", expression.as_str(), "tag_0", METRIC_NAME] { assert!(columns.contains(&column), "{query}: {columns:?}"); } assert!(!columns.contains(&"value"), "{query}: {columns:?}"); @@ -2522,17 +2533,23 @@ async fn native_zero_match_takes_the_layout_of_its_set_operand() { "the empty branch is not laid out by `narrow_total`" ); // The operation above pairs fields by position and keeps the shorter side's, which is the one - // field of `narrow_total`. + // field of `narrow_total`. The published layout names its values after the statement, so the + // answer is the one value column the shorter side brings, not the wider side's two. let output = plan .schema() .fields() .iter() .map(|field| field.name().as_str()) .collect::>(); - assert!( - output.contains(&"field_0") && !output.contains(&"field_1"), - "{output:?}" - ); + let values = plan + .schema() + .fields() + .iter() + .filter(|field| field.data_type() == &ArrowDataType::Float64) + .map(|field| field.name().as_str()) + .collect::>(); + let expression = parser::parse(query).unwrap().to_string(); + assert_eq!(vec![expression.as_str()], values, "{output:?}"); } /// Plans `query` with the native fused-series route on or off. @@ -2745,6 +2762,26 @@ fn find_fused_series(plan: &LogicalPlan) -> Option<&FusedSeries> { plan.inputs().into_iter().find_map(find_fused_series) } +/// The value column of a native producer's output. +/// +/// The native route names a value column after the expression that produced it, not after the +/// tree's field, so the sample kind is what identifies it: the one column typed `Float64` or the +/// canonical native histogram struct. Every other column is time or a tag. +fn native_value_field<'a>(fused: &'a FusedSeries, context: &str) -> &'a Field { + let mut values = fused.schema().fields().iter().filter(|field| { + field.data_type() == &ArrowDataType::Float64 + || field.data_type() == &native_histogram_value_type().as_arrow_type() + }); + let field = values + .next() + .unwrap_or_else(|| panic!("{context} published no field column")); + assert!( + values.next().is_none(), + "{context} published more than one field column" + ); + field +} + /// Timestamp over raw and evaluated vectors both use native execution, with anchors and offsets retained. #[tokio::test] async fn native_fused_timestamp_admits_raw_and_evaluated_vectors() { @@ -2903,10 +2940,12 @@ async fn native_fused_plans_dynamic_scalars() { } else { DEFAULT_TIME_INDEX_COLUMN }; + let expression = parser::parse(query).unwrap().to_string(); // A data-free `count_values` is a samples root, so its rows layout declares the per-row // name column behind the label the node writes. When the label it writes is the name - // itself, that one column is both: the layout names each column once. - let mut expected_schema = vec![expected_time, expected_value, expected_label]; + // itself, that one column is both: the layout names each column once. The value is named + // after the expression, as in every other rows layout. + let mut expected_schema = vec![expected_time, expression.as_str(), expected_label]; if expected_label != METRIC_NAME { expected_schema.push(METRIC_NAME); } @@ -3148,8 +3187,9 @@ async fn native_fused_rate_plans_to_a_fused_series_node() { let fused = find_fused_series(&plan).expect("the plan holds a native node"); assert_eq!("rate(some_metric[5m])", fused.promql()); assert_eq!( - // The rows layout declares the per-row name column behind the labels. - vec!["timestamp", "field_0", "tag_0", "__name__"], + // The value is named after the expression, and the rows layout declares the per-row name + // column behind the labels. + vec!["timestamp", "rate(some_metric[5m])", "tag_0", "__name__"], plan.schema() .fields() .iter() @@ -3283,6 +3323,57 @@ async fn native_fused_tag_partitioned_table_still_plans_fused_series() { assert!(plan.display_indent().to_string().contains("FusedSeries")); } +/// A destination label that names a value field is the statement's error, not a lost value. +/// +/// A tag is published under its own name, so a value field of that name has nowhere to go: the +/// projection used to skip the field, answering the statement without its value. Prometheus keeps +/// both, which one row cannot hold, so the collision is reported. +/// +/// The form whose regex does not match is the same collision reached from the other side: the label +/// the statement does not replace stays null, and the filter that keeps rows with a value then drops +/// the row, so a rename would answer no samples at all. +#[tokio::test] +async fn label_functions_reject_a_destination_label_that_names_a_value_field() { + for query in [ + // The overwrite: the destination label is the value field itself. + r#"label_replace(some_metric, "field_0", "oops", "tag_0", ".*")"#, + // The same destination label on the rows the regex leaves alone. + r#"label_replace(some_metric, "field_0", "oops", "tag_0", "does-not-match")"#, + // A missing source with a non-matching regex is a no-op before the rewrite is built. + r#"label_replace(some_metric, "field_0", "oops", "missing", "does-not-match")"#, + // An empty replacement for a new label is also a no-op before the rewrite is built. + r#"label_replace(some_metric, "field_0", "", "tag_0", ".*")"#, + r#"label_replace(some_metric, "field_0", "", "missing", ".*")"#, + r#"label_join(some_metric, "field_0", "-", "tag_0", "tag_0")"#, + ] { + let eval_stmt = build_eval_stmt(query); + let table_provider = build_test_table_provider( + &[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())], + 1, + 1, + ) + .await; + let error = PromPlanner::stmt_to_plan_with_annotations( + table_provider, + &eval_stmt, + &build_query_engine_state(), + None, + false, + ) + .await + .expect_err("a destination label cannot take a value field's name"); + let message = error.to_string(); + assert!( + message.contains("collides with a value field"), + "`{query}` reports the collision: {message}" + ); + assert!( + message.contains("field_0"), + "`{query}` names the colliding field: {message}" + ); + } +} + /// A label rewrite that collapses the partition column cannot be answered by one region. /// /// The canonical boundary's router only collects the pieces its own region's producer holds, so a @@ -3344,6 +3435,13 @@ async fn native_fused_label_rewrite_lowers_canonical_boundary() { sorted_column_names(&plan).contains(&"tag_0".to_string()), "the consumer retains the rewritten output label: {plan}" ); + let expression = parser::parse(query).unwrap().to_string(); + assert!( + sorted_column_names(&plan) + .iter() + .any(|name| name == &expression), + "the boundary publishes the expression-named value column `{expression}`: {plan}" + ); } /// Metric logical scans retain their table-id restriction on the direct fused scan input. @@ -3554,20 +3652,22 @@ async fn native_fused_metric_engine_two_aggregate_pair_is_one_execution() { (DEFAULT_SCHEMA_NAME.to_string(), "other_metric".to_string()), ]; let provider = || build_test_table_provider_with_tsid(&tables, 1, 1); - for (query, columns) in [ - ( - "sum(some_metric) / sum(other_metric)", - vec!["timestamp", "field_0", "__name__"], - ), - ( - "sum(sum(some_metric)) / sum(other_metric)", - vec!["timestamp", "field_0", "__name__"], - ), + for (query, tags) in [ + ("sum(some_metric) / sum(other_metric)", vec![]), + ("sum(sum(some_metric)) / sum(other_metric)", vec![]), ( "sum by (tag_0) (some_metric) / sum by (tag_0) (other_metric)", - vec!["timestamp", "field_0", "tag_0", "__name__"], + vec!["tag_0"], ), ] { + let expression = parser::parse(query).unwrap().to_string(); + // The value is named after the expression, and the rows layout declares the per-row name + // column last. + let columns = std::iter::once("timestamp") + .chain(std::iter::once(expression.as_str())) + .chain(tags) + .chain(std::iter::once("__name__")) + .collect::>(); let native = build_native_fused_plan_with_provider(query, true, provider().await).await; let fused = find_fused_series(&native) .unwrap_or_else(|| panic!("{query}: no fused node: {native}")); @@ -3579,7 +3679,7 @@ async fn native_fused_metric_engine_two_aggregate_pair_is_one_execution() { .iter() .map(|field| field.name().as_str()) .collect::>(), - "{query}: segmented output uses native tree names and order" + "{query}: segmented output uses the expression name" ); assert_eq!(2, fused.inputs().len(), "{query}: {native}"); assert!( @@ -4100,7 +4200,7 @@ async fn native_fused_composes_independent_histogram_binary_frames() { } assert!(!display.contains("Join:"), "{display}"); assert_eq!( - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names(query, &["tag_0", "__name__"]), sorted_column_names(&plan), "{display}" ); @@ -4129,14 +4229,15 @@ async fn native_composed_invalid_histogram_arithmetic_declares_float() { ); assert_eq!(2, fused.children().len(), "{display}"); assert!(!display.contains("Join:"), "{display}"); + let expression = parser::parse(query).unwrap().to_string(); let fields = plan.schema().fields(); let field = fields .iter() - .find(|field| field.name() == "field_0") - .unwrap(); + .find(|field| field.name() == &expression) + .unwrap_or_else(|| panic!("`{query}` names its value after the statement: {display}")); assert_eq!(field.data_type(), &ArrowDataType::Float64, "{display}"); assert_eq!( - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names(query, &["tag_0", "__name__"]), sorted_column_names(&plan), "{display}" ); @@ -4187,12 +4288,13 @@ async fn native_composed_binary_admits_fill() { ); } assert!(!display.contains("Join:"), "{display}"); + let expression = parser::parse(query).unwrap().to_string(); let field = plan .schema() .fields() .iter() - .find(|field| field.name() == "field_0") - .unwrap(); + .find(|field| field.name() == &expression) + .unwrap_or_else(|| panic!("`{query}` names its value after the statement: {display}")); assert_eq!( field.data_type(), &native_histogram_value_type().as_arrow_type(), @@ -4282,7 +4384,7 @@ async fn native_composed_binary_grouped_cardinality_keeps_many_side_labels() { // The many side keeps `tag_0` even though the pair ignores it: a grouping result holds one // series per series of that side, so the label is not narrowed away. assert_eq!( - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names(query, &["tag_0", "__name__"]), sorted_column_names(&plan), "{query}: {display}" ); @@ -4332,7 +4434,7 @@ async fn native_composed_binary_grouped_cardinality_keeps_many_side_labels() { "{display}" ); assert_eq!( - vec!["__name__", "field_0", "timestamp"], + expected_native_column_names("hist * ignoring(tag_0) floats", &["__name__"]), sorted_column_names(&plan), "{display}" ); @@ -4392,7 +4494,10 @@ async fn native_composed_binary_filled_one_side_carries_its_matching_labels() { assert!(!display.contains("Join:"), "{display}"); // The many side's `host` comes with the pairing and the one side's `job` with the filled step. assert_eq!( - vec!["__name__", "field_0", "host", "job", "timestamp"], + expected_native_column_names( + "hist * on(job) group_right fill_right(0) floats", + &["host", "job", "__name__"] + ), sorted_column_names(&plan), "{display}" ); @@ -4503,9 +4608,11 @@ async fn native_composed_binary_declares_only_include_labels_a_row_can_carry() { // The declared columns: the value and time columns plus the labels the walk names. The // per-row name column is declared on top of them, which // `native_rows_declare_the_name_of_each_row` pins. + let expression = parser::parse(query).unwrap().to_string(); let mut columns = sorted_column_names(&plan); - columns - .retain(|column| column != "field_0" && column != "timestamp" && column != "__name__"); + columns.retain(|column| { + column != &expression && column != "timestamp" && column != "__name__" + }); assert_eq!( declared .iter() @@ -4712,12 +4819,13 @@ async fn native_histogram_binary_fill_declares_possible_payload_kind() { panic!("expected native Binary for {query}: {display}"); }; assert!(!display.contains("Join:"), "{display}"); + let expression = parser::parse(query).unwrap().to_string(); let field = plan .schema() .fields() .iter() - .find(|field| field.name() == "field_0") - .unwrap(); + .find(|field| field.name() == &expression) + .unwrap_or_else(|| panic!("`{query}` names its value after the statement: {display}")); let expected = if histogram_output { native_histogram_value_type().as_arrow_type() } else { @@ -4754,11 +4862,25 @@ async fn native_fused_binary_inside_legacy_math_preserves_anchors() { ); assert_eq!(2, fused.children().len(), "{display}"); assert!(!display.contains("Join:"), "{display}"); - assert_eq!( - vec!["__name__", "ceil(field_0)", "tag_0", "timestamp"], - sorted_column_names(&plan), - "{display}" - ); + let expected_value = match query { + "ceil(clamp(histogram_count(hist), 1, 2) + on(tag_0) floats)" => { + "ceil(clamp(histogram_count(hist), 1, 2) + on (tag_0) floats)" + } + "ceil((clamp(histogram_count(hist @ end() offset 1m), 1, 2) + on(tag_0) floats @ start()))" => { + "ceil(clamp(histogram_count(hist @ end() offset 1m), 1, 2) + on (tag_0) floats @ start())" + } + _ => unreachable!("the test lists every query"), + }; + // The value is named after the statement, and the rows layout declares the per-row name + // column last. + let mut expected = vec![ + expected_value.to_string(), + "tag_0".to_string(), + "timestamp".to_string(), + "__name__".to_string(), + ]; + expected.sort(); + assert_eq!(expected, sorted_column_names(&plan), "{display}"); if query.contains("@ end()") { let left_tree = find_fused_series(&fused.children()[0]) .unwrap() @@ -4797,6 +4919,54 @@ async fn native_fused_subquery_window_enters_the_native_route() { } } +/// A matrix root publishes its value under the whole statement, not under the subquery's body. +/// +/// A subquery root is compiled by compiling its inner expression on the child grid, so the tree's +/// own field is named after that body. What the reader sees is the statement's layout, so the value +/// column keeps the full expression - `sum(some_metric)[5m:1m]`, not `sum(some_metric)` - and the +/// node's PromQL text, which is how a downstream stage reads those names back, is the same text. +#[tokio::test] +async fn native_matrix_root_names_its_value_after_the_statement() { + for query in [ + "sum(some_metric)[5m:1m]", + "sum by (tag_0) (rate(some_metric[2m]))[5m:1m]", + ] { + let table_provider = build_test_table_provider_with_options( + &[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())], + 1, + 1, + vec![], + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ConcreteDataType::float64_datatype(), + NATIVE_ENGINE, + ) + .await; + let plan = PromPlanner::stmt_to_plan_with_annotations( + table_provider, + &operator_eval_stmt(query), + &build_query_engine_state(), + None, + true, + ) + .await + .unwrap(); + let display = plan.display_indent().to_string(); + let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("`{query}`: {display}")); + let expression = parser::parse(query).unwrap().to_string(); + assert_eq!(expression, fused.promql(), "`{query}`: {display}"); + assert!( + fused + .schema() + .fields() + .iter() + .any(|field| field.name() == &expression), + "`{query}` publishes its value as the statement, not as the body: {}\n{display}", + fused.schema() + ); + } +} + #[tokio::test] async fn native_histogram_count_drops_float_input() { let query = "histogram_count(some_metric)"; @@ -4822,7 +4992,10 @@ async fn native_ceil_maps_raw_float_input() { .map(|field| field.name().clone()) .collect::>(); names.sort(); - assert_eq!(vec!["__name__", "field_0", "tag_0", "timestamp"], names); + assert_eq!( + vec!["__name__", "ceil(some_metric)", "tag_0", "timestamp"], + names + ); } /// Independent selector occurrences remain distinct scans in the native tree. @@ -4846,12 +5019,12 @@ async fn native_fused_binary_keeps_independent_scan_occurrences() { assert_eq!(2, fused.inputs().len(), "{query}"); if query.contains("on()") || query.contains("ignoring(tag_0)") { assert_eq!( - vec!["__name__", "field_0", "timestamp"], + expected_native_column_names(query, &["__name__"]), sorted_column_names(&plan) ); } else if query.contains("group_left") || query.contains("group_right") { assert_eq!( - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names(query, &["tag_0", "__name__"]), sorted_column_names(&plan) ); } @@ -5061,15 +5234,21 @@ async fn native_fused_cross_table_pair_reads_each_table_through_its_own_source() sources, "{query}" ); - // This pair route is not an aggregate route: it publishes the node's own field names, and - // every rows result names each row it publishes. A match that keeps only `on()`'s or - // `ignoring()`'s labels drops `tag_0`, and the name column is declared either way. - let expected = if query.contains("on()") || query.contains("ignoring(tag_0)") { - vec!["__name__", "field_0", "timestamp"] + // `or` reports the series of both sides, so its rows say each one's own name. Every rows + // layout declares the name column - the same column the walk's tags name when it holds one + // - so a match that keeps only `on()`'s or `ignoring()`'s labels still declares it. + let tags: &[&str] = if query.contains(" or ") { + &["__name__", "tag_0"] + } else if query.contains("on()") || query.contains("ignoring(tag_0)") { + &["__name__"] } else { - vec!["__name__", "field_0", "tag_0", "timestamp"] + &["tag_0", "__name__"] }; - assert_eq!(expected, sorted_column_names(&plan), "{query}"); + assert_eq!( + expected_native_column_names(query, tags), + sorted_column_names(&plan), + "{query}" + ); } } @@ -5300,26 +5479,57 @@ fn native_tables_provider(tables: &[(&str, &[&str], &str, &str)]) -> DfTableSour ) } -/// An `or` whose right table has a tag named like the left side's value field would report two -/// columns of one name, so it keeps the existing plan, which refuses that union with its own error; -/// a match that drops the tag stays native. +/// An `or` whose right table has a tag named like the left side's value field. +/// +/// The union reports one column per name, so the old raw field name collided with the right side's +/// tag and the existing plan refuses that union. The native route names its value column after the +/// expression, so the collision is gone: `field_0` stays a tag beside the expression's value. The +/// existing plan keeps refusing the same shape, unchanged. #[tokio::test] -async fn native_fused_cross_table_union_with_a_colliding_column_falls_back() { +async fn native_fused_cross_table_union_with_a_colliding_column_stays_native() { let eval_stmt = build_eval_stmt("some_metric or other_metric"); - let mut errors = Vec::new(); - for native_fused in [true, false] { - let error = PromPlanner::stmt_to_plan_with_annotations( - native_colliding_tag_provider().await, - &eval_stmt, - &build_query_engine_state(), - None, - native_fused, - ) - .await - .expect_err("the existing plan refuses a non-string union label"); - errors.push(error.to_string()); - } - assert_eq!(errors[0], errors[1]); + let error = PromPlanner::stmt_to_plan_with_annotations( + native_colliding_tag_provider().await, + &eval_stmt, + &build_query_engine_state(), + None, + false, + ) + .await + .expect_err("the existing plan refuses a non-string union label"); + assert!( + error.to_string().contains("field_0"), + "the existing plan reports the colliding column: {error}" + ); + + let plan = build_native_fused_plan_with_provider( + "some_metric or other_metric", + true, + native_colliding_tag_provider().await, + ) + .await; + let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("{plan}")); + assert_eq!( + expected_native_column_names( + "some_metric or other_metric", + &["tag_0", "field_0", "__name__"] + ), + sorted_column_names(&plan), + "{plan}" + ); + let field = native_value_field(fused, "some_metric or other_metric"); + assert_eq!(&ArrowDataType::Float64, field.data_type(), "{plan}"); + assert_eq!( + vec![ArrowDataType::Utf8; 2], + fused + .schema() + .fields() + .iter() + .filter(|field| matches!(field.name().as_str(), "tag_0" | "field_0")) + .map(|field| field.data_type().clone()) + .collect::>(), + "{plan}" + ); let plan = build_native_fused_plan_with_provider( "some_metric / on(tag_0) other_metric", @@ -5330,6 +5540,45 @@ async fn native_fused_cross_table_union_with_a_colliding_column_falls_back() { assert!(find_fused_series(&plan).is_some(), "{plan}"); } +#[tokio::test] +async fn native_fused_rejects_expression_names_that_collide_with_time_or_tag() { + for (table, tags, time, field, query, column) in [ + ("ts", Vec::<&str>::new(), "ts", "val", "ts", "ts"), + ("host", vec!["host"], "timestamp", "val", "host", "host"), + ] { + let provider = native_tables_provider(&[(table, &tags, time, field)]); + let error = PromPlanner::stmt_to_plan_with_annotations( + provider, + &build_eval_stmt(query), + &build_query_engine_state(), + None, + true, + ) + .await + .expect_err("the expression-derived value name collides with an output column"); + let message = error.to_string(); + // The collision is reported by name, not through the wrapped `DFSchema` refusal, whose + // display says only that building the plan failed. + assert!( + !message.contains("Internal error"), + "`{query}`: the error names the collision: {message}" + ); + assert!( + message.to_lowercase().contains("duplicate"), + "`{query}`: {message}" + ); + assert!( + message.contains(column), + "`{query}`: the error names the colliding column `{column}`: {message}" + ); + } + + // A wrapper gives the value a different expression name, so the same table remains native. + let provider = native_tables_provider(&[("ts", &[], "ts", "val")]); + let plan = build_native_fused_plan_with_provider("ts + 1", true, provider).await; + assert!(find_fused_series(&plan).is_some(), "{plan}"); +} + /// `up` and the info table `target_info`, whose data labels are `version` and `zone`. fn native_info_provider() -> DfTableSourceProvider { native_tables_provider(&[ @@ -5347,18 +5596,19 @@ fn native_info_provider() -> DfTableSourceProvider { /// data labels it can copy: every label of the info table, or only the ones the selector names. #[tokio::test] async fn native_fused_info_reads_the_info_table_after_the_base() { - for (query, columns) in [ + for (query, tags) in [ ( "info(up)", - vec![ - "__name__", "instance", "job", "ts", "val", "version", "zone", - ], + vec!["instance", "job", "version", "zone", "__name__"], ), ( "info(rate(up[5m]), {version=~\"v.+\"})", - vec!["__name__", "instance", "job", "ts", "val", "version"], + vec!["instance", "job", "version", "__name__"], ), ] { + let mut columns = vec![parser::parse(query).unwrap().to_string(), "ts".to_string()]; + columns.extend(tags.iter().map(|tag| (*tag).to_string())); + columns.sort(); let plan = build_native_fused_plan_with_provider(query, true, native_info_provider()).await; let fused = find_fused_series(&plan).unwrap_or_else(|| panic!("{query}: {plan}")); assert!( @@ -5411,15 +5661,21 @@ async fn native_fused_info_over_its_own_table_reads_two_scans_in_the_region() { crate::dist_plan::Commutativity::Commutative )); assert_eq!( - vec![ - "__name__", - "greptime_timestamp", - "greptime_value", - "instance", - "job", - "version", - "zone" - ], + { + // The value is named after the expression, and the rows layout declares the per-row + // name column on top of the labels the info node reports. + let mut names = vec![ + "greptime_timestamp".to_string(), + parser::parse(query).unwrap().to_string(), + "instance".to_string(), + "job".to_string(), + "version".to_string(), + "zone".to_string(), + "__name__".to_string(), + ]; + names.sort(); + names + }, sorted_column_names(&plan) ); } @@ -5569,7 +5825,7 @@ async fn native_fused_three_selector_trees_bind_every_occurrence() { "{query}" ); // All four trees take the segmented route, including same-table selectors. The boundary - // publishes the native root's names directly rather than adding a naming projection. + // publishes the expression name directly rather than adding a naming projection. assert!( matches!(plan, LogicalPlan::Extension(_)), "{query}: expected the segmented native boundary: {plan}" @@ -5583,9 +5839,9 @@ async fn native_fused_three_selector_trees_bind_every_occurrence() { "{query}: expected a segmented producer: {plan}" ); assert_eq!( - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names(query, &["tag_0", "__name__"]), sorted_column_names(&plan), - "{query}: the segmented boundary uses the native tree's names" + "{query}: the segmented boundary uses the expression name" ); } } @@ -5646,9 +5902,18 @@ fn sorted_column_names(plan: &LogicalPlan) -> Vec { names } +fn expected_native_column_names(query: &str, tags: &[&str]) -> Vec { + let mut names = vec![ + parser::parse(query).unwrap().to_string(), + "timestamp".to_string(), + ]; + names.extend(tags.iter().map(|tag| (*tag).to_string())); + names.sort(); + names +} + /// An aggregate is split: the merge stays on the frontend and the per-series prefix runs in the -/// producer. Native names follow the compiled tree; grouping labels, values and timestamps retain -/// their existing schema roles. +/// producer. Grouping labels, values and timestamps retain their existing schema roles. #[tokio::test] async fn native_fused_aggregate_splits_into_a_final_over_a_partial_producer() { for query in [ @@ -5747,67 +6012,27 @@ async fn native_fused_aggregate_splits_into_a_final_over_a_partial_producer() { !partial_display.contains(" complete") && !partial_display.contains(" final"), "`{query}`: {partial_display}" ); - // Final and the statement boundary both publish native field names for physical conversion. + // The tree retains field identity while the boundary names its value after the expression. let NativeType::Samples(schema) = &final_node.tree().root().output else { panic!("`{query}`: final root does not produce samples"); }; - for field in schema.fields.iter() { - assert!( - final_node - .schema() - .fields() - .iter() - .any(|column| column.name() == field.name.as_ref()), - "`{query}`: final output is missing native field {}", - field.name - ); - } + assert_eq!("field_0", schema.fields[0].name.as_ref()); assert_eq!( plan.schema(), final_node.schema(), "`{query}` native output schema" ); - let expected_names: &[&str] = match query { - "sum(some_metric)" => &["__name__", "field_0", "timestamp"], - "avg(some_metric)" => &["__name__", "field_0", "timestamp"], - "count(some_metric)" => &["__name__", "field_0", "timestamp"], - "min(some_metric)" => &["__name__", "field_0", "timestamp"], - "max(some_metric)" => &["__name__", "field_0", "timestamp"], - "group(some_metric)" => &["__name__", "field_0", "timestamp"], - "stddev(some_metric)" => &["__name__", "field_0", "timestamp"], - "stdvar(some_metric)" => &["__name__", "field_0", "timestamp"], - "sum by (tag_0) (rate(some_metric[5m]))" => { - &["__name__", "field_0", "tag_0", "timestamp"] + let tags: &[&str] = match query { + "sum by (tag_0) (rate(some_metric[5m]))" => &["tag_0", "__name__"], + r#"sum without (tag_0) (label_replace(some_metric, "dst", "$1", "tag_0", "(.*)"))"# + | r#"sum by (dst) (label_replace(some_metric, "dst", "$1", "tag_0", "(.*)"))"# => { + &["dst", "__name__"] } - "sum without (tag_0) (some_metric)" => &["__name__", "field_0", "timestamp"], - r#"sum without (tag_0) (label_replace(some_metric, "dst", "$1", "tag_0", "(.*)"))"# => { - &["__name__", "dst", "field_0", "timestamp"] - } - r#"sum by (dst) (label_replace(some_metric, "dst", "$1", "tag_0", "(.*)"))"# => { - &["__name__", "dst", "field_0", "timestamp"] - } - "sum(rate(some_metric[5m]))" => &["__name__", "field_0", "timestamp"], - "ceil(sum(some_metric))" => &["__name__", "field_0", "timestamp"], - "sum(avg(some_metric) by (tag_0))" => &["__name__", "field_0", "timestamp"], - "sum(8 * rate(some_metric[5m]))" => &["__name__", "field_0", "timestamp"], - "max(10 - rate(some_metric[5m]))" => &["__name__", "field_0", "timestamp"], - "100 - avg(rate(some_metric[5m]))" => &["__name__", "field_0", "timestamp"], - "sum(some_metric) * 2" => &["__name__", "field_0", "timestamp"], - "100 / sum(some_metric)" => &["__name__", "field_0", "timestamp"], - "-sum(some_metric)" => &["__name__", "field_0", "timestamp"], - "abs(sum(some_metric)) * 2" => &["__name__", "field_0", "timestamp"], - - "histogram_quantile(time() / 150, sum by (le) (rate(some_metric[5m])))" => { - &["__name__", "field_0", "timestamp"] - } - "histogram_fraction(time() / 150, time() / 75 + 0.5, sum by (le) (rate(some_metric[5m])))" => { - &["__name__", "field_0", "timestamp"] - } - _ => panic!("missing exact native schema expectation for `{query}`"), + _ => &["__name__"], }; assert_eq!( - expected_names, - sorted_column_names(&plan).as_slice(), + expected_native_column_names(query, tags), + sorted_column_names(&plan), "exact native output columns for `{query}`" ); } @@ -5966,7 +6191,7 @@ async fn native_fused_scalar_broadcast_keeps_segmentation() { ); } -/// The split boundary takes its ordered names from the Final tree, without an aliasing plan. +/// The split boundary publishes expression names without an aliasing plan. #[tokio::test] async fn native_split_boundary_uses_native_tree_column_names() { let query = "sum by (tag_0) (abs(some_metric))"; @@ -5992,9 +6217,13 @@ async fn native_split_boundary_uses_native_tree_column_names() { .map(|field| field.name().as_str()) .collect::>(); // The Final node is a rows layout, so it declares the per-row name column behind the label the - // grouping keeps; the value field keeps its position between the time index and the labels. - assert_eq!(vec!["timestamp", "field_0", "tag_0", "__name__"], names); - assert_eq!(&names[1..1 + field_names.len()], field_names); + // grouping keeps; the value between the time index and the labels is named after the + // expression. + let expression = parser::parse(query).unwrap().to_string(); + assert_eq!( + vec!["timestamp", expression.as_str(), "tag_0", "__name__"], + names + ); assert!( datatypes::schema::is_metric_name_field( plan.schema() @@ -6055,7 +6284,7 @@ async fn native_split_and_complete_aggregate_produce_identical_values() { let final_node = find_final_split(&split).expect("the query must use a split aggregate"); let partial = partial_below(final_node).unwrap(); let complete = FusedSeries::try_new( - Arc::from(query), + Arc::from(stmt.expr.to_string()), complete_tree, final_node.binding().clone(), partial.children().to_vec(), @@ -6127,7 +6356,7 @@ async fn native_split_and_complete_aggregate_produce_identical_values() { .downcast_ref::() .unwrap(); let values = batch - .column_by_name("field_0") + .column_by_name(&stmt.expr.to_string()) .unwrap() .as_any() .downcast_ref::() @@ -6325,8 +6554,8 @@ async fn native_fused_multi_source_segments_bind_their_own_table_scan() { ); } -/// Nested aggregates and their wrappers use the native root's field names, including segmented -/// trees that the aggregate split declines. +/// Nested aggregates and their wrappers publish expression names, including segmented trees that +/// the aggregate split declines. #[tokio::test] async fn native_fused_nested_rate_abs_aggregates_use_native_schema_names() { for query in [ @@ -6341,9 +6570,9 @@ async fn native_fused_nested_rate_abs_aggregates_use_native_schema_names() { panic!("{query}: native planning must produce a fused-series node: {native}") }); assert_eq!( - vec!["__name__", "field_0", "timestamp"], + expected_native_column_names(query, &["__name__"]), sorted_column_names(&native), - "{query}: the boundary uses the native tree's field name" + "{query}: the boundary uses the expression name" ); assert!( native @@ -6361,7 +6590,7 @@ async fn native_fused_nested_rate_abs_aggregates_use_native_schema_names() { } } -/// A nested aggregate declines the split but its segmented output still uses the native root name. +/// A nested aggregate declines the split but its segmented output still uses the expression name. #[tokio::test] async fn native_fused_declined_nested_aggregate_uses_native_name() { let query = "sum(avg(some_metric) by (tag_0))"; @@ -6369,15 +6598,15 @@ async fn native_fused_declined_nested_aggregate_uses_native_name() { assert!(find_final_split(&plan).is_none(), "{query}: {plan}"); assert_eq!( - vec!["__name__", "field_0", "timestamp"], + expected_native_column_names(query, &["__name__"]), sorted_column_names(&plan), - "a bare outer sum drops the inner grouping label and names the value after the compiled root field, not legacy expression text" + "a bare outer sum drops the inner grouping label and names the value after the expression" ); let grouped = build_native_fused_plan("sum by (tag_0) (some_metric)", true).await; assert_eq!( sorted_column_names(&grouped), - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names("sum by (tag_0) (some_metric)", &["tag_0", "__name__"]), "an explicit by grouping keeps its label in the exact native schema" ); } @@ -6877,7 +7106,10 @@ async fn native_fused_clamp_aggregate_is_admitted() { .map(|field| field.name().clone()) .collect::>(); native_names.sort(); - assert_eq!(vec!["__name__", "field_0", "timestamp"], native_names); + assert_eq!( + expected_native_column_names(query, &["__name__"]), + native_names + ); let legacy_provider = make_provider(); let legacy_result = PromPlanner::stmt_to_plan_with_annotations( @@ -7036,6 +7268,20 @@ async fn native_metric_discovery_name_rewrite_canonicalizes_by_name() { ); } +/// A propagated matcher re-plans a scalar operand of the other side natively, and the native +/// output name includes the matcher it now carries. The outer legacy binary must read the field +/// columns the re-planned operand publishes, not the ones captured before propagation. +#[tokio::test] +async fn native_fused_rewritten_scalar_operand_uses_rewritten_field_columns() { + let query = r#"(some_metric * 2) / on(tag_0) some_metric{tag_0="x",__field__="field_0"}"#; + let plan = build_native_fused_plan(query, true).await; + let display = plan.display_indent().to_string(); + assert!( + display.contains(r#"some_metric{tag_0="x"} * 2"#), + "the re-planned scalar operand publishes its rewritten expression name: {display}" + ); +} + #[tokio::test] async fn native_fused_preparation_decline_preserves_legacy_plan() { let query = "absent(vector(1)) + on(tag_0) some_metric{tag_0=\"x\",__field__=\"field_0\"}"; @@ -9473,8 +9719,9 @@ async fn or_operator_with_unknown_metric_does_not_require_tsid() { /// as a column. Without it every discovered metric falls into one group. #[tokio::test] async fn native_sum_by_name_separates_discovered_metrics() { + let query = "sum by (__name__) ({tag_0=\"x\"})"; let plan = build_native_fused_plan_with_provider( - "sum by (__name__) ({tag_0=\"x\"})", + query, true, metric_discovery_provider(&["alpha_total", "beta_total"]).await, ) @@ -9491,9 +9738,11 @@ async fn native_sum_by_name_separates_discovered_metrics() { fused.inputs().len(), "the discovered metrics share one input: {plan}" ); - // The grouping keeps the name, so the aggregate answers one group per metric rather than one. + // The grouping keeps the name, so the aggregate answers one group per metric rather than one, + // and the value column is named after the statement's expression. + let expression = parser::parse(query).unwrap().to_string(); assert_eq!( - vec!["timestamp", "field_0", "__name__"], + vec!["timestamp", expression.as_str(), "__name__"], plan.schema() .fields() .iter() @@ -9933,10 +10182,7 @@ async fn a_label_rewrite_keeps_a_float_answer_on_the_route() { let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan) .unwrap_or_else(|| panic!("{query} left the route instead of answering: {display}")); - let field = fused - .schema() - .field_with_unqualified_name(greptime_native_histogram()) - .unwrap_or_else(|_| panic!("{query} published no field column: {display}")); + let field = native_value_field(fused, &format!("{query}: {display}")); assert_eq!( &ArrowDataType::Float64, field.data_type(), @@ -9984,10 +10230,7 @@ async fn a_segmented_parent_answers_the_kind_of_the_uncut_root() { let fused = find_fused_series(&plan).unwrap_or_else(|| { panic!("{query} left the native route instead of segmenting: {display}") }); - let field = fused - .schema() - .field_with_unqualified_name(greptime_native_histogram()) - .unwrap_or_else(|_| panic!("{query} published no field column: {display}")); + let field = native_value_field(fused, &format!("{query}: {display}")); let expected = if histogram { native_histogram_value_type().as_arrow_type() } else { @@ -10010,10 +10253,7 @@ async fn native_histogram_selector_plans_a_native_producer() { assert!(!display.contains("HistogramFold:"), "{display}"); let fused = find_fused_series(&plan).expect("the statement plans to a native producer"); // The output field column is the canonical native histogram struct. - let field = fused - .schema() - .field_with_unqualified_name(greptime_native_histogram()) - .expect("the producer reports the histogram field"); + let field = native_value_field(fused, "some_metric"); assert_eq!( &native_histogram_value_type().as_arrow_type(), field.data_type(), @@ -10098,10 +10338,9 @@ async fn presence_and_order_aggregates_fold_histograms_natively() { let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan) .unwrap_or_else(|| panic!("{query} left the route instead of counting: {display}")); - let field = fused - .schema() - .field_with_unqualified_name(greptime_native_histogram()) - .unwrap_or_else(|_| panic!("{query} published no field column: {display}")); + // The published value column is named after the expression, not after the compiled field, + // so the field is the plan's one value column. + let field = native_value_field(fused, query); assert_eq!( &ArrowDataType::Float64, field.data_type(), @@ -10168,10 +10407,7 @@ async fn native_histogram_instant_selection_keeps_the_histogram_column() { build_native_fused_plan_with_provider("some_metric offset 1m", true, table_provider).await; let display = plan.display_indent_schema().to_string(); let fused = find_fused_series(&plan).expect("the statement plans to a native producer"); - let field = fused - .schema() - .field_with_unqualified_name(greptime_native_histogram()) - .expect("the producer reports the histogram field"); + let field = native_value_field(fused, "some_metric offset 1m"); assert_eq!( &native_histogram_value_type().as_arrow_type(), field.data_type(), @@ -12848,6 +13084,35 @@ async fn test_count_values_expr() { assert_eq!(plan.display_indent_schema().to_string(), expected); } +/// An alias may name an `Int64` value `__tsid`: only the internal identifier is stripped. +/// +/// The series identifier is the internal `UInt64` key or the `Utf8` label synthesized from it, so +/// those two are the only `__tsid` columns the planner removes. `count(...)` answers `Int64`, and an +/// alias that publishes it as `__tsid` is the statement's own column: stripping it would answer the +/// statement with no value at all. +#[tokio::test] +async fn test_tsid_alias_keeps_an_int64_value() { + let eval_stmt = + QueryLanguageParser::apply_alias_extension(build_eval_stmt("count(some_metric)"), "__tsid"); + let table_provider = build_test_table_provider( + &[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())], + 1, + 1, + ) + .await; + let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + let display = plan.display_indent_schema().to_string(); + let field = plan + .schema() + .fields() + .iter() + .find(|field| field.name() == "__tsid") + .unwrap_or_else(|| panic!("the aliased value keeps its column: {display}")); + assert_eq!(&ArrowDataType::Int64, field.data_type(), "{display}"); +} + #[tokio::test] async fn test_value_alias() { let mut eval_stmt = EvalStmt { @@ -15091,8 +15356,8 @@ async fn native_fused_metric_engine_count_values_output_labels() { // The rows layout declares the per-row name column on top of the labels the walk names. expected.push("__name__".to_string()); expected.push("timestamp".to_string()); - // The value column keeps the compiled tree's field name rather than the expression text. - expected.push("field_0".to_string()); + // The value column is named after the expression that produced it. + expected.push(parser::parse(query).unwrap().to_string()); expected.sort(); assert_eq!(sorted_column_names(&plan), expected, "{query}"); } @@ -15158,7 +15423,7 @@ async fn native_composed_binary_nests_as_one_fused_tree() { ); assert_eq!(1, right.children().len(), "{query}: {display}"); assert_eq!( - vec!["__name__", "field_0", "tag_0", "timestamp"], + expected_native_column_names(query, &["tag_0", "__name__"]), sorted_column_names(&plan), "{query}: {display}" ); diff --git a/tests-integration/src/tests/native_aggregate_test.rs b/tests-integration/src/tests/native_aggregate_test.rs index ecc84f6f023..325190c5e93 100644 --- a/tests-integration/src/tests/native_aggregate_test.rs +++ b/tests-integration/src/tests/native_aggregate_test.rs @@ -87,6 +87,7 @@ use promql::vector_match::MatchingLabels; use query::datafusion::QUERY_PARALLELISM_HINT; use query::dist_plan::MergeScanExec; use query::options::QueryOptions; +use query::parser::{PromQuery, QueryLanguageParser, QueryStatement}; use query::physical_wrapper::{PhysicalPlanWrapper, PhysicalPlanWrapperRef}; use servers::query_handler::sql::SqlQueryHandler; use session::context::{QueryContext, QueryContextRef}; @@ -430,9 +431,6 @@ struct AggregateCase { partial_state: Option<&'static str>, /// Whether this case's global aggregate has an explicitly specified single owner. single_owner: bool, - /// Native output schema. The native aggregate preserves its source field name (`v`), rather - /// than the legacy planner's computed-expression name. - native_value_column: &'static str, } /// Builds one case from a PromQL expression and its `(labels, value)` rows. @@ -454,7 +452,6 @@ fn case(promql: String, expected: &[(&[(&str, &str)], f64)], per_partition: bool root_values: None, partial_state: None, single_owner: false, - native_value_column: "v", } } @@ -485,6 +482,19 @@ fn tql_eval(promql: &str) -> String { format!("TQL EVAL ({EVAL_SECS}, {EVAL_SECS}, '1m') {promql}") } +fn expression_name(promql: &str) -> String { + let query = PromQuery { + query: promql.to_string(), + ..PromQuery::default() + }; + let QueryStatement::Promql(stmt, _) = + QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap() + else { + unreachable!() + }; + stmt.expr.to_string() +} + /// Runs `promql` at [`EVAL_SECS`] and returns its rows with the native nodes its plans held. async fn run_query( instance: &Arc, @@ -1192,12 +1202,12 @@ fn assert_rows(actual: &[ResultRow], expected: &[ResultRow], context: &str) { } /// Asserts the native boundary's ordered names and types, independently of legacy names. -fn assert_native_column_layout(native: &QueryResult, tags: &[&str], query: &str) { - let mut expected = vec!["ts", "v"]; - expected.extend_from_slice(tags); +fn assert_native_column_layout(native: &QueryResult, tags: &[&str], promql: &str, query: &str) { + let mut expected = vec!["ts".to_string(), expression_name(promql)]; + expected.extend(tags.iter().map(|tag| (*tag).to_string())); // The rows layout declares the per-row name column after the labels; an aggregate computes a // value of its own, so that column is null here. - expected.push("__name__"); + expected.push("__name__".to_string()); assert_eq!( expected, native.columns, "`{query}`: native boundary columns" @@ -1281,7 +1291,7 @@ fn assert_native_columns( ); assert_eq!( expected_value_column, native.value_column, - "`{query}`: native value column must use the compiled root field name" + "`{query}`: native value column must use the statement expression" ); assert_eq!( 1, @@ -1774,7 +1784,7 @@ async fn check_two_source_segregation( legacy.columns, "`{query}`: the legacy plan names the value column after its expression" ); - assert_native_column_layout(&native, &[], &query); + assert_native_column_layout(&native, &[], promql, &query); assert_rows( &native.rows, &legacy.rows, @@ -1883,7 +1893,7 @@ async fn check_nested_aggregate_case( assert_rows(&legacy.rows, &case.expected, &format!("legacy `{query}`")); let (native, capture) = run_query(instance, &case.promql, true, observer).await; - assert_native_column_layout(&native, &[], &query); + assert_native_column_layout(&native, &[], &case.promql, &query); assert_rows( &native.rows, &legacy.rows, @@ -1916,7 +1926,7 @@ async fn check_case(instance: &Arc, observer: &NativePlanObserver, cas .keys() .map(String::as_str) .collect::>(); - assert_native_column_layout(&native, &tags, &query); + assert_native_column_layout(&native, &tags, promql, &query); if promql.starts_with("count ") { assert!( legacy.column_types.contains(&DataType::Int64), @@ -1928,8 +1938,8 @@ async fn check_case(instance: &Arc, observer: &NativePlanObserver, cas &legacy.rows, &format!("native `{query}` vs legacy"), ); - assert_native_schema(&native, case.native_value_column, &case.expected, &query); - assert_native_columns(&native, &legacy, &query, "v"); + assert_native_schema(&native, &expression_name(promql), &case.expected, &query); + assert_native_columns(&native, &legacy, &query, &expression_name(promql)); assert_rows(&native.rows, &case.expected, &format!("native `{query}`")); assert_eq!( @@ -2190,7 +2200,7 @@ async fn native_aggregate_topk_over_global_partitions() { let (legacy, legacy_capture) = run(false).await; assert!(legacy_capture.fused.is_empty() && legacy_capture.finalize.is_empty()); let (native, capture) = run(true).await; - assert_native_columns(&native, &legacy, &sql, "v"); + assert_native_columns(&native, &legacy, &sql, &expression_name(&promql)); assert_rows( &native.rows, &legacy.rows, @@ -2225,7 +2235,7 @@ async fn native_aggregate_topk_over_global_partitions() { } } expected.sort_by(|a, b| (a.ts_millis, &a.labels).cmp(&(b.ts_millis, &b.labels))); - assert_native_schema(&native, "v", &expected, &sql); + assert_native_schema(&native, &expression_name(&promql), &expected, &sql); assert_rows(&native.rows, &expected, &format!("expected `{sql}`")); // `topk` above a global aggregate is not split, so the scan side folds per series and the // ranking runs above it. A plan may hold several fused nodes (the frontend root and the region @@ -2295,9 +2305,13 @@ async fn native_aggregate_clamp_executes_on_both_routes() { close_enough(native.rows[0].value.unwrap(), 10.0), "native `{query}` value" ); - assert_eq!("v", native.value_column, "native `{query}` value column"); - // The rows layout: the time index, the one value, and the declared name column, which an - // aggregate leaves null. + assert_eq!( + expression_name(&promql), + native.value_column, + "native `{query}` value column" + ); + // The rows layout: the time index, the one expression-named value, and the declared name + // column, which an aggregate leaves null. assert_eq!(3, native.columns.len(), "native `{query}` columns"); assert_eq!("__name__", native.columns[2], "native `{query}` columns"); @@ -2781,9 +2795,9 @@ async fn native_histogram_over_the_cut_splits_and_matches_legacy() { // reports over the complete input, labels included - is compared before the plan shape, so a // shape regression cannot hide it. let tags: &[&str] = if *single_owner { &[] } else { &["job"] }; - assert_native_column_layout(&native, tags, &query); - assert_native_schema(&native, "v", &legacy.rows, &query); - assert_native_columns(&native, &legacy, &query, "v"); + assert_native_column_layout(&native, tags, promql, &query); + assert_native_schema(&native, &expression_name(promql), &legacy.rows, &query); + assert_native_columns(&native, &legacy, &query, &expression_name(promql)); assert_rows( &native.rows, &legacy.rows, @@ -3032,15 +3046,19 @@ async fn native_bare_selector_aggregate_value_columns() { let mark = observer.mark(); let (native_names, native_types, native_rows, capture) = read(&instance, &observer, &promql, true).await; + let expression = expression_name(&promql); + let mut expected_names = vec!["ts".to_string()]; + match expected_values { + 1 => expected_names.push(expression), + 2 => expected_names.extend([format!("{expression}.a"), format!("{expression}.b")]), + _ => unreachable!(), + } + if !single_owner { + expected_names.push("job".to_string()); + } // The rows layout declares the per-row name column after the labels; every statement in // this case is an aggregate, which computes its value and leaves that column null. - let expected_names = match (expected_values, single_owner) { - (1, true) => vec!["ts", "v", "__name__"], - (1, false) => vec!["ts", "v", "job", "__name__"], - (2, true) => vec!["ts", "a", "b", "__name__"], - (2, false) => vec!["ts", "a", "b", "job", "__name__"], - _ => unreachable!(), - }; + expected_names.push("__name__".to_string()); assert_eq!( expected_names, native_names, "`{query}` native boundary columns" diff --git a/tests-integration/src/tests/promql_test.rs b/tests-integration/src/tests/promql_test.rs index f3be25c3ddd..b8cb400c55a 100644 --- a/tests-integration/src/tests/promql_test.rs +++ b/tests-integration/src/tests/promql_test.rs @@ -5256,11 +5256,15 @@ async fn distributed_native_promql_zero_match_or_real() { ); let (columns, rows) = native_labelled_rows(output).await; - assert_eq!( - columns, - vec!["__name__", "host", "ts", "val"], - "{expression}" - ); + // The value is named after the statement's expression, not `val`. + let mut expected_columns = vec![ + "__name__".to_string(), + "host".to_string(), + "ts".to_string(), + expression_name(expression), + ]; + expected_columns.sort(); + assert_eq!(columns, expected_columns, "{expression}"); let actual = rows .iter() .map(|(time, labels, value)| { @@ -5454,7 +5458,7 @@ async fn distributed_native_promql_metric_discovery_reconciles_mixed_ordering() .clone() }); let values = batch - .column_by_name("val") + .column_by_name(&expression_name(expression)) .expect("one value column") .as_any() .downcast_ref::() @@ -5626,9 +5630,7 @@ async fn data_free_rows(output: Output) -> (Vec, Vec<(i64, f64)>) { (columns, rows) } -/// A statement that reads no table plans natively over the wire, where there is no region to place -/// anything in: the frontend evaluates one fused node, so the answer keeps the native column name -/// instead of the one the existing planner derives from an expression. +/// A statement that reads no table plans natively over the wire, with an expression-named value. #[tokio::test(flavor = "multi_thread")] async fn distributed_native_promql_data_free_statement_plans_natively() { common_telemetry::init_default_ut_logging(); @@ -5644,15 +5646,15 @@ async fn distributed_native_promql_data_free_statement_plans_natively() { .build(false) .await; let instance = cluster.fe_instance().clone(); - for (expression, expected_columns, expected_rows) in [ + for (expression, labels, expected_rows) in [ // `start`/`end` are seconds, so this one-step range evaluates at 3000ms. - ("time()", &["time", "value"][..], &[(3000, 3.0)][..]), + ("time()", &[][..], &[(3000, 3.0)][..]), ( // A samples root declares the name of each row it publishes, and `vector(1)` has // none, so the column is present and null. `time()` above is a scalar root and keeps // two columns. r#"label_replace(vector(1), "host", "host1", "", "")"#, - &["time", "value", "host", "__name__"][..], + &["host", "__name__"][..], &[(3000, 1.0)][..], ), ] { @@ -5679,11 +5681,17 @@ async fn distributed_native_promql_data_free_statement_plans_natively() { "a data-free statement reads no region: {plan:?}" ); let (columns, rows) = data_free_rows(output).await; + let QueryStatement::Promql(stmt, _) = + QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap() + else { + unreachable!() + }; + let mut expected_columns = vec!["time".to_string(), stmt.expr.to_string()]; + expected_columns.extend(labels.iter().map(|label| (*label).to_string())); assert_eq!(expected_columns, columns, "{expression}"); assert_eq!(expected_rows, rows.as_slice(), "{expression}"); - // The same statement through the existing planner answers the same rows; only the native - // route names the value column after the tree rather than after an expression. + // The same statement through the existing planner answers the same rows. let legacy = instance .do_promql_query(&query, native_query_context(false, 2)) .await @@ -6805,9 +6813,20 @@ async fn native_promql_sort_absent_and_histogram_quantile_match_legacy() { let legacy_rows = legacy.iter().map(|batch| batch.num_rows()).sum::(); assert_eq!(native_rows, legacy_rows); assert!(native_rows >= 1, "histogram_quantile must produce data"); - // The rows root lays its columns out as time, values, labels, and the name last. + let query = PromQuery { + query: shape.to_string(), + ..PromQuery::default() + }; + let QueryStatement::Promql(stmt, _) = + QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap() + else { + unreachable!() + }; + let expression = stmt.expr.to_string(); + // The rows root lays its columns out as time, the expression-named value, the labels, + // and the name last. assert_eq!( - vec!["ts", "val", "host", "__name__"], + vec!["ts", expression.as_str(), "host", "__name__"], native_columns .iter() .map(|column| column.name.as_str()) @@ -6902,15 +6921,132 @@ async fn native_promql_sort_absent_and_histogram_quantile_match_legacy() { // keep the metric name; `absent` answers a value of its own for a label set it derives // from the matcher, and Prometheus' `absent` leaves the name out of that set. The // existing route has no name column at all, so the column is checked against what the - // shape answers and left out of the comparison. + // shape answers and left out of the comparison below. + // + // Both routes answer the same rows; only the one value column's header differs, + // because the existing planner keeps the tree's field name while the native route + // names that column after the expression. Compare what the columns carry - the + // timestamp, the label set and the value, in response order - and the native naming. let reported = if shape.starts_with("absent") { ReportedName::Nothing } else { ReportedName::Metric("native_shapes") }; - let native = named_rows_text(native, reported).await; - let legacy = named_rows_text(legacy, ReportedName::NoColumn).await; - assert_eq!(native, legacy, "{shape}; SQL: {sql}"); + let collect = |output: Output| async move { + match output.data { + OutputData::Stream(stream) => collect_batches(stream).await.unwrap(), + OutputData::RecordBatches(batches) => batches, + _ => unreachable!("query returns record batches"), + } + }; + let rows = |batches: &common_recordbatch::RecordBatches, reported: ReportedName<'_>| { + batches + .iter() + .flat_map(|batch| { + let batch = batch.df_record_batch(); + let schema = batch.schema(); + // The name column is checked against what the shape answers and kept out of + // the label set, so the rows of the two routes compare whole. + checked_name_column(schema.fields(), batch.columns(), reported); + let value_index = schema + .fields() + .iter() + .position(|field| field.data_type() == &DataType::Float64) + .expect("one float value column"); + let timestamps = batch + .column_by_name("ts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let labels = schema + .fields() + .iter() + .filter(|field| { + !matches!( + field.data_type(), + DataType::Float64 | DataType::Timestamp(..) + ) && field.name() != "__name__" + }) + .map(|field| { + let column = cast( + batch.column_by_name(field.name()).unwrap().as_ref(), + &DataType::Utf8, + ) + .unwrap(); + ( + field.name().clone(), + column + .as_any() + .downcast_ref::() + .unwrap() + .clone(), + ) + }) + .collect::>(); + let values = + cast(batch.column(value_index).as_ref(), &DataType::Float64).unwrap(); + let values = values.as_any().downcast_ref::().unwrap(); + (0..batch.num_rows()) + .map(|row| { + ( + timestamps.value(row), + labels + .iter() + .filter(|(_, column)| { + column.is_valid(row) && !column.value(row).is_empty() + }) + .map(|(name, column)| { + (name.clone(), column.value(row).to_string()) + }) + .collect::>(), + values.value(row), + ) + }) + .collect::>() + }) + .collect::>() + }; + let native = collect(native).await; + let legacy = collect(legacy).await; + let native_names = native + .schema() + .column_schemas() + .iter() + .map(|column| column.name.clone()) + .collect::>(); + let native_rows = rows(&native, reported); + let legacy_rows = rows(&legacy, ReportedName::NoColumn); + assert_eq!(native_rows, legacy_rows, "{shape}; SQL: {sql}"); + if shape.starts_with("sort") { + assert!(!native_rows.is_empty(), "{shape} must produce data: {sql}"); + } + let legacy_labels = legacy + .schema() + .column_schemas() + .iter() + .filter(|column| { + !column.data_type.is_timestamp() + && column.data_type + != datatypes::prelude::ConcreteDataType::float64_datatype() + }) + .map(|column| column.name.clone()) + .collect::>(); + // The native answer's columns are the time index, the expression-named value, the + // labels the existing route also declares, and the name column it appends last. + let mut expected_names = vec!["ts".to_string(), expression_name(shape)]; + expected_names.extend(legacy_labels); + expected_names.push("__name__".to_string()); + assert_eq!( + native_names, expected_names, + "{shape}: native boundary columns" + ); + assert!( + native_names[2..native_names.len() - 1] + .iter() + .all(|name| matches!(name.as_str(), "host" | "le")), + "{shape}: native boundary columns: {native_names:?}" + ); } } } @@ -7541,11 +7677,10 @@ async fn native_kind_rows(output: Output) -> Vec<(i64, String, String, f64)> { let hosts = hosts.as_any().downcast_ref::().unwrap(); let kinds = kinds.as_any().downcast_ref::().unwrap(); let values = batch - .column_by_name("val") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); + .columns() + .iter() + .find_map(|column| column.as_any().downcast_ref::()) + .expect("one value column"); for row in 0..batch.num_rows() { rows.push(( timestamps.value(row), @@ -8157,8 +8292,15 @@ async fn native_promql_zero_match_or_real_tql() { // A rows answer reports the statement's own columns, the per-row name column among them. // `alpha_total` reads one named metric, and `or` reports each side with the label set it // read, so every row here is a series of that metric and keeps its name while the empty - // side adds nothing. - assert_eq!(columns, vec!["__name__", "host", "ts", "val"], "{query}"); + // side adds nothing. The value is named after the statement's expression, not `val`. + let mut expected_columns = vec![ + "__name__".to_string(), + "host".to_string(), + "ts".to_string(), + expression_name(query), + ]; + expected_columns.sort(); + assert_eq!(columns, expected_columns, "{query}"); let actual = rows .iter() .map(|(time, labels, value)| { @@ -8499,6 +8641,19 @@ async fn discovery_query(instance: &Arc, promql: &str) -> Output { .unwrap() } +fn expression_name(promql: &str) -> String { + let query = PromQuery { + query: promql.to_string(), + ..PromQuery::default() + }; + let QueryStatement::Promql(stmt, _) = + QueryLanguageParser::parse_promql(&query, &QueryContext::arc()).unwrap() + else { + unreachable!() + }; + stmt.expr.to_string() +} + /// `(time, labels, value)` of every row, sorted, whatever the columns are named: the time index is /// the timestamp column, the value the one `Float64` column, and every other column a label that a /// NULL or empty value leaves out. Also returns the column names. @@ -8860,7 +9015,18 @@ async fn native_promql_cross_table_pairs_over_different_schemas() { 2, "{context}: expected one distinct SeriesScanExec per selector occurrence" ); - assert_eq!(native_columns, columns, "{context}"); + let mut expected_columns = columns + .into_iter() + .map(|column| { + if column == "val" { + expression_name(query) + } else { + column.to_string() + } + }) + .collect::>(); + expected_columns.sort(); + assert_eq!(native_columns, expected_columns, "{context}"); assert_eq!(native.len(), expected.len(), "{context}: {native:?}"); for (actual, expected) in native.iter().zip(&expected) { assert_eq!( @@ -9055,7 +9221,27 @@ async fn native_promql_three_selector_trees_bind_every_occurrence() { .map(|field| field.name().clone()) .collect::>(); plan_columns.sort(); - assert_eq!(plan_columns, columns, "{query}"); + let expression_query = PromQuery { + query: query.to_string(), + ..PromQuery::default() + }; + let QueryStatement::Promql(stmt, _) = + QueryLanguageParser::parse_promql(&expression_query, &QueryContext::arc()).unwrap() + else { + unreachable!() + }; + let mut expected_columns = columns + .into_iter() + .map(|column| { + if column == "val" { + stmt.expr.to_string() + } else { + column.to_string() + } + }) + .collect::>(); + expected_columns.sort(); + assert_eq!(plan_columns, expected_columns, "{query}"); assert_eq!(native, expected, "{query}"); let legacy = native_result_rows(native_tql_output(&instance, &sql, false).await).await; assert_eq!(native, legacy, "legacy: {query}"); @@ -9278,7 +9464,18 @@ async fn native_promql_info_enriches_from_the_info_table() { assert_eq!(plan.name(), "FusedSeriesExec", "{query}"); assert_eq!(plan.children().len(), 2, "{query}"); assert_eq!(observer.take_raw_sources().len(), 2, "{query}"); - assert_eq!(native_columns, columns, "{query}"); + let mut expected_columns = columns + .into_iter() + .map(|column| { + if column == "val" { + expression_name(query) + } else { + column.to_string() + } + }) + .collect::>(); + expected_columns.sort(); + assert_eq!(native_columns, expected_columns, "{query}"); assert_eq!(native, expected, "{query}"); // The existing plan refuses `info`: "Unsupported expr type: info" for one argument, and @@ -10085,10 +10282,18 @@ async fn native_hybrid_binary_empty_on_grouped_matching_and_errors() { native_labelled_rows(native_tql_output(&instance, &sql, true).await).await; let plan = observer.two_input_fused(); assert_eq!(plan.name(), "FusedSeriesExec", "{query}"); - assert_eq!( - columns, - vec!["__name__", "host", "kind", "ts", "val", "zone_name"] - ); + // The value is named after the statement's expression, and the rows answer reports the + // per-row name column among its labels; a math kernel drops the name, so it holds no value. + let mut expected_columns = vec![ + "__name__".to_string(), + "host".to_string(), + "kind".to_string(), + "ts".to_string(), + expression_name(query), + "zone_name".to_string(), + ]; + expected_columns.sort(); + assert_eq!(columns, expected_columns); assert_eq!(rows, expected, "{query}"); let (_, legacy) = native_labelled_rows(native_tql_output(&instance, &sql, false).await).await; @@ -10582,6 +10787,26 @@ async fn native_promql_metric_two_aggregate_pair_matches_legacy() { .iter() .map(|column| column.name.as_str()) .collect::>(); + let expression_query = PromQuery { + query: query.to_string(), + ..PromQuery::default() + }; + let QueryStatement::Promql(stmt, _) = + QueryLanguageParser::parse_promql(&expression_query, &QueryContext::arc()).unwrap() + else { + unreachable!() + }; + let expression = stmt.expr.to_string(); + let expected_columns = expected_columns + .into_iter() + .map(|column| { + if column == "val" { + expression.as_str() + } else { + column + } + }) + .collect::>(); assert_eq!( expected_columns, boundary_columns, "{query}: native boundary columns" @@ -10657,15 +10882,13 @@ async fn native_promql_metric_two_aggregate_pair_matches_legacy() { // The columns both routes report: the time index, one value column, and one column per // label. The labels themselves are compared row by row above, and the time index is `ts` on - // both. The value column's *name* is not equal, and that is this route's existing naming for - // a computed value column rather than anything this shape introduces: the node names its - // fields after the tree's own root schema, while the existing planner names a computed column - // after its expression (the same difference is pinned for a cross-table pair by - // `native_fused_cross_table_pair_reads_each_table_through_its_own_source`). So the - // shape is asserted, and the value column's name is pinned as the raw field it comes from. - // The rows layout declares the per-row name column and this statement has none to report: - // a division computes a value of its own and its inputs are aggregates, so every row's name - // is null. The column is therefore the only difference from the existing route's schema. + // both. Compare time and label columns independently: the native boundary names that value + // column after the canonical expression, where the existing planner names a computed column + // after its expression too, and the two texts differ in the spaces a PromQL parse and a + // legacy rewrite produce. The rows layout additionally declares the per-row name column and + // this statement has none to report: a division computes a value of its own and its inputs + // are aggregates, so every row's name is null. That column is therefore the one difference + // from the existing route's schema. assert_eq!( native_columns.len(), legacy_columns.len() + 1, @@ -11710,13 +11933,19 @@ async fn native_promql_rewrites_the_label_count_values_writes() { ); } -/// `topk`/`bottomk` key their groups by series labels, while the existing plan also accepts a value -/// field there and forms one group per sample value. A `by()` therefore keeps every -/// group's winner, and the native route has to report the same rows: the union below it reports its -/// series, and the window above it partitions by the same value column. The rows are what this -/// asserts - one winner per value group - rather than that no native node answers the union. +/// `topk`/`bottomk` key their groups by series labels, and `by (val)` names no label a series +/// carries: `val` is the value field, not a label. Prometheus therefore puts every series of the +/// input into one empty group and answers only that group's winner, so the statement returns one +/// row rather than one per sample value. +/// +/// Both routes are about that grouping, but only the native one answers Prometheus here. The value +/// column is named after the expression, so a `by (val)` over it selects nothing and the one empty +/// group keeps the global winner. The existing plan still partitions its window by the `val` column +/// it publishes, one group per sample value; that divergence is known and is not pinned here. What +/// the native route must keep is the answer, with the union below it running as one native node +/// rather than a legacy join. #[tokio::test(flavor = "multi_thread")] -async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() { +async fn native_promql_ranks_a_value_field_grouping_as_one_group() { common_telemetry::init_default_ut_logging(); let (standalone, observer) = native_test_instance(GreptimeDbStandaloneBuilder::new( "native_promql_ranking_value_field", @@ -11748,28 +11977,25 @@ async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() { .await; let pair = r#"native_rank_demo{host="x"} or native_rank_other{host="y"}"#; - // `by (val)` forms two groups of one sample each, so both series survive: two rows, one winner - // per value group, on both routes. - for query in [ - format!("topk by (val) (1, {pair})"), - format!("bottomk by (val) (1, {pair})"), + // No series carries a `val` label, so the grouping is empty: every series lands in that one + // group and the answer is its winner, `y` at 8 for `topk` and `x` at 2 for `bottomk`. + for (query, expected) in [ + ( + format!("topk by (val) (1, {pair})"), + vec![(0, "y".to_string(), 8.0)], + ), + ( + format!("bottomk by (val) (1, {pair})"), + vec![(0, "x".to_string(), 2.0)], + ), ] { let sql = format!("TQL EVAL (0, 0, '1s') {query}"); - observer.clear(); - let legacy_rows = native_result_rows(native_tql_output(&instance, &sql, false).await).await; - assert_eq!( - legacy_rows, - vec![(0, "x".to_string(), 2.0), (0, "y".to_string(), 8.0)], - "{query}: the existing plan keeps one row per value group" - ); - observer.clear(); let native_rows = native_result_rows(native_tql_output(&instance, &sql, true).await).await; let root = datafusion::physical_plan::displayable(observer.root_plan().as_ref()) .indent(true) .to_string(); - // The union has to stay native rather than become the legacy join, and it must not fold the - // two value groups into one. + // The union has to stay native rather than become the legacy join. assert!( observer.fused().is_some(), "{query}: the union runs as one fused node: {root}" @@ -11778,7 +12004,7 @@ async fn native_promql_keeps_a_ranking_grouped_by_a_value_field() { !root.contains("JoinExec"), "{query}: the union is not a legacy join: {root}" ); - assert_eq!(native_rows, legacy_rows, "{query}: full rows"); + assert_eq!(native_rows, expected, "{query}: full rows"); } // A label grouping is what the native node does honor, so it still runs natively and keeps every @@ -12090,7 +12316,16 @@ async fn native_promql_delta_temporality_rate_and_increase_match_legacy() { observer.clear(); let (columns, rows) = native_labelled_rows(native_tql_output(&instance, &sql, true).await).await; - assert_eq!(columns, vec!["__name__", "host", "ts", "val"]); + // The value is named after the statement's expression, not `val`, and the rows answer + // reports the per-row name column among its labels; a division drops the name. + let mut expected_columns = vec![ + "__name__".to_string(), + "host".to_string(), + "ts".to_string(), + expression_name(&query), + ]; + expected_columns.sort(); + assert_eq!(columns, expected_columns, "{query}"); assert_eq!(rows.len(), 1, "{query}: {rows:?}"); assert_eq!(rows[0].0, 120_000, "{query}"); assert_eq!( @@ -12792,9 +13027,10 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() { // `count_values` names a label no scan carries, so the statement keeps a plan that reports it. let sql = "TQL EVAL (0, 0, '1s') count_values(\"bucket\", vector(1))"; // The native route answers `count_values` itself now, so this is its own answer: one series at - // t=0 whose generated label is "1" and whose count is 1. The aggregate drops the metric name - - // as Prometheus does - so the declared name column is there and null, and it is the one column - // the existing planner's answer does not have; the two headers differ while the numbers agree. + // t=0 whose generated label is "1" and whose count is 1. The value column is named after the + // expression, and the aggregate drops the metric name - as Prometheus does - so the declared + // name column is there and null; it is the one column the existing planner's answer does not + // have, and is left out of the columns compared below. let native = named_rows_text( native_tql_output(&instance, sql, true).await, ReportedName::Nothing, @@ -12802,11 +13038,11 @@ async fn native_promql_dynamic_scalars_match_prometheus_and_legacy() { .await; assert_eq!( native, - "+---------------------+-------+--------+\ - \n| time | value | bucket |\ - \n+---------------------+-------+--------+\ - \n| 1970-01-01T00:00:00 | 1.0 | 1 |\ - \n+---------------------+-------+--------+" + "+---------------------+-----------------------------------+--------+\ + \n| time | count_values(\"bucket\", vector(1)) | bucket |\ + \n+---------------------+-----------------------------------+--------+\ + \n| 1970-01-01T00:00:00 | 1.0 | 1 |\ + \n+---------------------+-----------------------------------+--------+" ); } @@ -13618,10 +13854,16 @@ async fn native_promql_recursive_matching_segments_match_legacy() { expected, native, "matching on host drops job from output labels" ); - assert_eq!( - vec!["__name__", "host", "ts", "val"], - columns.iter().map(String::as_str).collect::>() - ); + // The value is named after the statement's expression, and the rows answer reports the + // per-row name column among its labels. + let mut expected_columns = vec![ + "__name__".to_string(), + "host".to_string(), + "ts".to_string(), + expression_name(query), + ]; + expected_columns.sort(); + assert_eq!(expected_columns, columns); } /// Segments are cut where the physical planner would otherwise have to gather a per-series prefix. @@ -14199,18 +14441,17 @@ async fn native_promql_real_query_q07_throttled_periods() { .iter() .all(|debug| debug.contains("FusedSeries")) ); - // Known computed-column naming gap from the ungrouped two-source case; values agree. + // The two routes name the computed value column differently - the existing planner after its + // own rewrite, the native route after the expression - and the labelled rows compared above + // are what both report. assert!( legacy_columns .iter() .any(|column| column.contains("prom_increase")) ); - assert_eq!( - native_columns - .iter() - .filter(|column| *column == "val") - .count(), - 1 + assert!( + native_columns.contains(&expression_name(query)), + "Q07 native value column: {native_columns:?}" ); } @@ -14316,14 +14557,13 @@ async fn native_promql_real_query_q04_namespace_cpu_requests() { .iter() .all(|debug| debug.contains("FusedSeries")) ); - // Known computed-column naming gap from the ungrouped two-source case; values agree. + // The two routes name the computed value column differently - the existing planner after its + // own rewrite, the native route after the expression - and the labelled rows compared above + // are what both report. assert!(legacy_columns.iter().any(|column| column.contains("max("))); - assert_eq!( - native_columns - .iter() - .filter(|column| *column == "val") - .count(), - 1 + assert!( + native_columns.contains(&expression_name(query)), + "Q04 native value column: {native_columns:?}" ); } @@ -14668,11 +14908,20 @@ async fn native_promql_dynamic_topk_matches_legacy() { "a bare topk above no aggregate runs natively: {bare_sql}" ); // The same ranking answers the same rows on both routes; the rows keep the name of the series - // `topk` selected, which this route states in its own declared column. - let mut named_columns = bare_legacy.0.clone(); - named_columns.push("__name__".to_string()); - named_columns.sort(); - assert_eq!(bare_native.0, named_columns, "{bare_sql}: columns"); + // `topk` selected, which this route states in its own declared column. The two routes name the + // one value column differently - the existing planner keeps the tree's `val`, the native route + // names it after the expression - so the columns are the existing answer's, with that name and + // the name column. + let mut expected_columns = bare_legacy + .0 + .iter() + .filter(|column| *column != "val") + .cloned() + .collect::>(); + expected_columns.push("__name__".to_string()); + expected_columns.push(expression_name("topk(3, native_dyn_topk)")); + expected_columns.sort(); + assert_eq!(bare_native.0, expected_columns, "{bare_sql}: columns"); assert_eq!( bare_native.1, named_rows(bare_legacy.1, "native_dyn_topk"), diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 9556774d2a4..c5353da192f 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1881,19 +1881,17 @@ fn result_columns(schema: &SchemaRef) -> Vec<(String, DataType, bool)> { /// /// The tags are stated by the caller rather than read off the result: a vector that matches no /// series reports no row to read them from, and still answers with its tag columns. -fn assert_native_result_schema(schema: &SchemaRef, tags: &[&str], context: &str) { - // The time index and the one field the tree reports, then one `Utf8` column per tag. +fn assert_native_result_schema(schema: &SchemaRef, tags: &[&str], value_name: &str, context: &str) { + // The time index and the one field the tree reports, then one `Utf8` column per tag. The + // native route names the value column after the expression, so the caller states that name; + // `value`/`val` are the legacy route's names. let mut columns = vec![ ( if tags.is_empty() { "time" } else { "ts" }.to_string(), DataType::Timestamp(TimeUnit::Millisecond, None), false, ), - ( - if tags.is_empty() { "value" } else { "val" }.to_string(), - DataType::Float64, - true, - ), + (value_name.to_string(), DataType::Float64, true), ]; for name in tags { columns.push(((*name).to_string(), DataType::Utf8, true)); @@ -2218,9 +2216,13 @@ async fn assert_raw_scan_fused_formats(observer: &NativeFusedObserver, frontend: .as_any() .downcast_ref::() .unwrap(); + // The native route names its value column after the expression, so the + // type is what identifies it; `val` is only the legacy route's name. let values = batch - .column_by_name("val") - .unwrap() + .columns() + .iter() + .find(|column| column.data_type() == &DataType::Float64) + .expect("one Float64 value column") .as_any() .downcast_ref::() .unwrap(); @@ -2744,7 +2746,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { }; let (plan, schema, rows) = promql_output(frontend, query, true, start, end).await; assert_rows_eq(&rows, &expected, query); - assert_native_result_schema(&schema, tags, query); + assert_native_result_schema(&schema, tags, query, query); let fused = assert_native_plan(&plan, scan_partitions, query); assert_eq!( fused["metrics"]["native.root.output_values"].as_u64(), @@ -2769,13 +2771,33 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { ); assert_rows_eq(&legacy_rows, &expected, query); assert_legacy_result_schema(&legacy_schema, tags, query); - // Both constant cases preserve the legacy time/value schema; computed vector columns are - // compared by role. + // Both routes report the time column and one Float64 value. The native route names that + // value after the expression, so the constant cases agree by role, not by header name. if scan_partitions.is_none() { assert_eq!( - result_columns(&legacy_schema), - result_columns(&schema), - "`{query}`: the two routes must report the same constant schema" + 2, + schema.fields().len(), + "`{query}`: native constant schema" + ); + assert_eq!( + 2, + legacy_schema.fields().len(), + "`{query}`: legacy constant schema" + ); + assert_eq!( + schema.field(0).data_type(), + legacy_schema.field(0).data_type(), + "`{query}`: time column type" + ); + assert_eq!( + &DataType::Float64, + schema.field(1).data_type(), + "`{query}`: native value type" + ); + assert_eq!( + &DataType::Float64, + legacy_schema.field(1).data_type(), + "`{query}`: legacy value type" ); } } @@ -2788,7 +2810,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) { observer.clear(); let (plan, schema, rows) = promql_output(frontend, query, true, "0", "2").await; assert!(rows.is_empty(), "`{query}` reported {} rows", rows.len()); - assert_native_result_schema(&schema, &["host"], query); + assert_native_result_schema(&schema, &["host"], query, query); assert!( executes_fused_series(&plan), "`{query}` did not execute the fused route"