diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs index 5741ac94a82..90b3fdf8a9e 100644 --- a/src/query/src/planner.rs +++ b/src/query/src/planner.rs @@ -49,6 +49,7 @@ use sql::statements::query::Query; use sql::statements::statement::Statement; use sql::statements::tql::Tql; use sqlparser::ast::{AccessExpr, Value, visit_expressions_mut}; +use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME; use crate::datafusion::QUERY_FALLBACK_HINT; use crate::dist_plan::DistPlannerOptions; @@ -82,6 +83,9 @@ pub trait LogicalPlanner: Send + Sync { /// A remote native plan may place fused evaluation only above independent raw sources. /// In particular, a bare scan under FusedSeries would require shipping the process-local /// fused plan to a region, which this gate must not admit. +/// +/// A fused node that reads no region is the one exception: it has nothing to place in a region, so +/// the frontend evaluates it and the transport this gate protects is never used. fn remote_native_raw_plan_supported(plan: &LogicalPlan) -> bool { fn walk(plan: &LogicalPlan, found_raw: &mut bool) -> bool { if matches!(plan, LogicalPlan::TableScan(_)) { @@ -102,7 +106,81 @@ fn remote_native_raw_plan_supported(plan: &LogicalPlan) -> bool { plan.inputs().iter().all(|child| walk(child, found_raw)) } let mut found_raw = false; - walk(plan, &mut found_raw) && found_raw + walk(plan, &mut found_raw) && (found_raw || is_regionless_fused_plan(plan)) +} + +/// Whether `plan` is a native fused plan with no region below it. +/// +/// Two shapes qualify, and both are already the finished plan: a statement that reads no data - +/// `time()`, `vector(1)`, a label rewrite over one of them - is one node with no input at all, +/// and a discovered selector that matched no metric table is planned over the typed empty frame +/// [`is_empty_discovered_input`](crate::promql::planner::native::is_empty_discovered_input) +/// builds. Neither has a scan to ship, so neither needs the remote transport. +/// +/// The route stacks its own nodes above such an input: a split aggregate runs a `Final` half over +/// the `Partial` half that reads it, and a segmented statement runs its parent over the segment's +/// own node. So the inputs a node reads are followed through fused nodes only - never through +/// another extension, and never through a scan - and every leaf that is not one of them rejects +/// the shape. +fn is_regionless_fused_plan(plan: &LogicalPlan) -> bool { + // `strip_tsid_column` publishes a statement's own columns over a node that reported the + // internal series id, which a data-free statement can synthesize with `label_replace`. That + // wrapper is this route's own, so it is read through when it drops exactly that one column. + if let LogicalPlan::Projection(projection) = plan { + return strips_only_tsid(plan, projection.input.as_ref()) + && is_regionless_fused_chain(projection.input.as_ref()); + } + is_regionless_fused_chain(plan) +} + +/// Whether `plan` is a fused node whose inputs are fused nodes or the typed empty frame. +fn is_regionless_fused_chain(plan: &LogicalPlan) -> bool { + let LogicalPlan::Extension(extension) = plan else { + return false; + }; + let Some(fused) = extension + .node + .as_any() + .downcast_ref::() + else { + return false; + }; + fused.children().iter().all(|child| { + crate::promql::planner::native::is_empty_discovered_input(child) + || is_regionless_fused_chain(child) + }) +} + +/// Whether `plan` is the projection that drops exactly the internal series-id column of `inner`. +/// +/// That is the projection `strip_tsid_column` builds: one plain column per field of its input, +/// in order, except `__tsid__`. A computed expression, a reordering or a second dropped column is +/// some other plan and is not read through. +fn strips_only_tsid(plan: &LogicalPlan, inner: &LogicalPlan) -> bool { + let LogicalPlan::Projection(projection) = plan else { + return false; + }; + if projection + .expr + .iter() + .any(|expr| !matches!(expr, DfExpr::Column(_))) + { + return false; + } + let stripped = inner + .schema() + .fields() + .iter() + .filter(|field| field.name() != DATA_SCHEMA_TSID_COLUMN_NAME) + .map(|field| field.name()) + .collect::>(); + let projected = plan + .schema() + .fields() + .iter() + .map(|field| field.name()) + .collect::>(); + projected == stripped && projected.len() + 1 == inner.schema().fields().len() } pub struct DfLogicalPlanner { @@ -405,8 +483,9 @@ impl DfLogicalPlanner { .map_err(BoxedError::new) .context(QueryPlanSnafu)?; - // Only per-selector raw sources have a supported remote transport. Decide while the - // original statement is still available: physical planning cannot fall back safely. + // Only per-selector raw sources have a supported remote transport, and a statement that + // reads no table needs none. Decide while the original statement is still available: + // physical planning cannot fall back safely. let plan = if native_requested && !self.engine_state.in_process_region_transport() && !remote_native_raw_plan_supported(&plan) @@ -831,10 +910,16 @@ mod tests { use catalog::RegisterTableRequest; use catalog::memory::MemoryCatalogManager; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; + use common_query::OutputData; + use common_recordbatch::RecordBatch; + use common_recordbatch::util::collect_batches; use common_time::Timezone; + use datatypes::arrow::array::{Array, Float64Array, StringArray}; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema}; - use session::context::{QueryContext, QueryContextBuilder}; + use session::context::{ + MetricDiscoveryScope, QueryContext, QueryContextBuilder, QueryContextRef, + }; use store_api::metric_engine_consts::{ DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, @@ -1187,6 +1272,372 @@ mod tests { } } + /// Plans `query` on the fixture engine with the native route requested. The engine reaches its + /// regions over the wire, which is the route the admission gate guards. + async fn plan_remote_native_promql( + engine: &QueryEngineRef, + query_ctx: &QueryContextRef, + query: &str, + ) -> LogicalPlan { + let stmt = QueryLanguageParser::parse_promql( + &PromQuery { + query: query.to_string(), + start: "0".to_string(), + end: "10".to_string(), + step: "5s".to_string(), + lookback: "300s".to_string(), + alias: None, + }, + query_ctx, + ) + .unwrap(); + engine + .planner() + .plan(&stmt, query_ctx.clone()) + .await + .unwrap() + } + + /// A query context that asks for the native route and states what a nameless selector may + /// discover; the default context declines discovery altogether. + fn native_query_ctx(scope: MetricDiscoveryScope) -> QueryContextRef { + let mut ctx = (*QueryContext::arc()).clone(); + ctx.set_metric_discovery_scope(scope); + ctx.set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), "true".to_string()); + Arc::new(ctx) + } + + fn native_discovery_query_ctx() -> QueryContextRef { + native_query_ctx(MetricDiscoveryScope::Unrestricted) + } + + /// The engine's physical plan for `plan` and the batches it answers. + async fn execute_plan( + engine: &QueryEngineRef, + plan: LogicalPlan, + query_ctx: &QueryContextRef, + ) -> ( + Arc, + Vec, + ) { + let output = engine.execute(plan, query_ctx.clone()).await.unwrap(); + let physical = output + .meta + .plan + .expect("the engine reports its physical plan"); + let batches = match output.data { + OutputData::Stream(stream) => collect_batches(stream).await.unwrap(), + OutputData::RecordBatches(batches) => batches, + other => panic!("a PromQL query answers rows, not {other:?}"), + } + .take(); + (physical, batches) + } + + /// Every node name in the tree rooted at `root`, the root included. + fn physical_plan_names( + root: &Arc, + ) -> Vec { + let mut names = Vec::new(); + let mut stack = vec![Arc::clone(root)]; + while let Some(node) = stack.pop() { + names.push(node.name().to_string()); + stack.extend(node.children().into_iter().cloned()); + } + names + } + + /// Every logical plan node below `root`, the root included. + fn logical_plan_nodes(root: &LogicalPlan) -> Vec<&LogicalPlan> { + let mut nodes = Vec::new(); + let mut stack = vec![root]; + while let Some(node) = stack.pop() { + nodes.push(node); + stack.extend(node.inputs()); + } + nodes + } + + fn fused_series_nodes(plan: &LogicalPlan) -> Vec<&promql::extension_plan::FusedSeries> { + logical_plan_nodes(plan) + .into_iter() + .filter_map(|node| match node { + LogicalPlan::Extension(extension) => extension + .node + .as_any() + .downcast_ref::( + ), + _ => None, + }) + .collect() + } + + fn numeric_values(batches: &[RecordBatch], column: &str) -> Vec { + batches + .iter() + .flat_map(|batch| { + batch + .column_by_name(column) + .unwrap_or_else(|| panic!("no `{column}` column")) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect() + } + + fn label_values(batches: &[RecordBatch], column: &str) -> Vec { + batches + .iter() + .flat_map(|batch| { + let array = batch + .column_by_name(column) + .unwrap_or_else(|| panic!("no `{column}` column")) + .as_any() + .downcast_ref::() + .expect("a label column is Utf8"); + (0..array.len()) + .map(|row| array.value(row).to_string()) + .collect::>() + }) + .collect() + } + + /// Asserts `plan` is a native plan that reads no region, and that every fused node in it binds + /// no metric: a data-free or zero-match statement has no table to name. + fn assert_regionless_native_plan(plan: &LogicalPlan, query: &str) { + let display = plan.display_indent().to_string(); + assert!(display.contains("FusedSeries"), "{query}: {display}"); + for absent in ["TableScan", "MergeScan", "RawSeriesSource"] { + assert!( + !display.contains(absent), + "{query} contains {absent}: {display}" + ); + } + let fused = fused_series_nodes(plan); + assert!(!fused.is_empty(), "{query}: {display}"); + for node in fused { + assert_eq!( + None, + node.binding().metric_name, + "{query} bound a metric to an input that reads no table: {display}" + ); + } + } + + /// Asserts every empty input under `plan` declares the typed empty raw series frame: the + /// planner's own definition of that input, and the frame a scan would have declared, so the + /// physical planner reads it as a frame producer instead of looking for a scan below it. + fn assert_typed_empty_inputs(plan: &LogicalPlan, query: &str) { + let empty = logical_plan_nodes(plan) + .into_iter() + .filter(|node| matches!(node, LogicalPlan::EmptyRelation(_))) + .collect::>(); + assert!(!empty.is_empty(), "{query} has no empty input"); + for node in empty { + assert!( + crate::promql::planner::native::is_empty_discovered_input(node), + "{query}: {node:?}" + ); + let LogicalPlan::EmptyRelation(relation) = node else { + unreachable!(); + }; + assert!(!relation.produce_one_row, "{query}: {relation:?}"); + let schema = relation.schema.as_arrow(); + for (key, expected) in [ + ("greptime.series.content", "raw"), + ("greptime.series.labels", "explicit"), + ("greptime.series.ordering", "contiguous"), + ] { + assert_eq!( + Some(expected), + schema.metadata().get(key).map(String::as_str), + "{query}: {key}" + ); + } + // The one value field the empty input is bound to, named as the route names a value. + let fields = schema.field_with_name("fields").expect("a series frame"); + let arrow_schema::DataType::Struct(payloads) = fields.data_type() else { + panic!("{query}: {fields:?}"); + }; + assert_eq!( + vec!["value"], + payloads + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + "{query}" + ); + } + } + + /// Asserts the physical plan of a regionless native statement: the fused node runs on the + /// frontend and no region read is planned. + fn assert_regionless_physical_plan( + physical: &Arc, + query: &str, + ) { + let names = physical_plan_names(physical); + assert!( + names.iter().any(|name| name == "FusedSeriesExec"), + "{query} is evaluated on the frontend by the fused node: {physical:?}" + ); + assert!( + !names.iter().any(|name| name == "MergeScanExec"), + "{query} reads no region: {physical:?}" + ); + } + + /// A statement that reads no table plans natively over the wire, where there is no region to + /// place anything in: the frontend evaluates the fused node under the native column names + /// rather than the ones the existing planner derives from an expression. + #[tokio::test] + async fn test_remote_native_promql_plans_a_data_free_statement_natively() { + let engine = create_promql_test_engine_with( + crate::options::QueryOptions { + experimental_native_promql: true, + ..Default::default() + }, + false, + ); + let query_ctx = native_discovery_query_ctx(); + for (query, expected_columns, expected_values, expected_labels) in [ + // `time()` answers the evaluation grid in seconds. + ( + "time()", + &["time", "value"][..], + &[0.0, 5.0, 10.0][..], + &[][..], + ), + ( + "vector(1)", + &["time", "value"][..], + &[1.0, 1.0, 1.0][..], + &[][..], + ), + ( + r#"label_replace(vector(1), "host", "host1", "", "")"#, + &["time", "value", "host"][..], + &[1.0, 1.0, 1.0][..], + &[("host", "host1")][..], + ), + // A synthesized `__tsid` is published under the statement's own columns: the stripping + // projection above the node is read through rather than mistaken for a remote plan. + ( + r#"label_replace(vector(1), "__tsid", "x", "", "")"#, + &["time", "value"][..], + &[1.0, 1.0, 1.0][..], + &[][..], + ), + ] { + let plan = plan_remote_native_promql(&engine, &query_ctx, query).await; + let display = plan.display_indent().to_string(); + assert_regionless_native_plan(&plan, query); + assert!( + !display.contains("greptime_value"), + "{query} keeps the native value column: {display}" + ); + // A data-free statement has no input at all, so nothing is scanned or left empty. + assert_eq!( + expected_columns, + plan.schema() + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + "{query}: {display}" + ); + let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await; + assert_regionless_physical_plan(&physical, query); + assert_eq!( + expected_values, + numeric_values(&batches, "value").as_slice(), + "{query}: {batches:?}" + ); + for (label, value) in expected_labels { + assert_eq!( + vec![value.to_string(); batches_row_count(&batches)], + label_values(&batches, label), + "{query}: label `{label}`: {batches:?}" + ); + } + } + } + + /// The rows every batch of `batches` holds. + fn batches_row_count(batches: &[RecordBatch]) -> usize { + batches.iter().map(|batch| batch.num_rows()).sum() + } + + /// A nameless selector that matches no metric table has no region to read either, so the typed + /// empty input it is planned as stays that input on both routes and the node above it is + /// evaluated on the frontend. Binding a metric to it is what used to turn this statement into + /// an internal error over the wire. + #[tokio::test] + async fn test_remote_native_promql_plans_a_zero_match_selector_natively() { + let engine = create_promql_test_engine_with( + crate::options::QueryOptions { + experimental_native_promql: true, + ..Default::default() + }, + false, + ); + let query_ctx = native_discovery_query_ctx(); + for (query, expected_rows, expected_labels) in [ + // The selection itself, over the typed empty frame. + (r#"{job="api"}"#, 0, &[][..]), + // A whole-vector operation over it, which the route plans as a segment: its parent node + // reads the segment's own fused node. + (r#"sort({job="api"})"#, 0, &[][..]), + // A fold splits into a `Final` half over the `Partial` half that reads the input. + (r#"sum({job="api"})"#, 0, &[][..]), + // `absent` answers one series per evaluation point over zero input, labelled by the + // selector's matchers. The grid holds three points. + (r#"absent({job="api"})"#, 3, &[("job", "api")][..]), + ] { + let plan = plan_remote_native_promql(&engine, &query_ctx, query).await; + let display = plan.display_indent().to_string(); + assert_regionless_native_plan(&plan, query); + assert!( + display.contains("EmptyRelation"), + "{query} keeps the typed empty input: {display}" + ); + assert_typed_empty_inputs(&plan, query); + let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await; + assert_regionless_physical_plan(&physical, query); + // No table matched, so the answer is the empty vector, not an error. + assert_eq!( + expected_rows, + batches_row_count(&batches), + "{query} answered rows over a selector that matches no metric: {physical:?}" + ); + for (label, value) in expected_labels { + assert_eq!( + vec![value.to_string(); expected_rows], + label_values(&batches, label), + "{query}: label `{label}`: {batches:?}" + ); + } + } + + // An allowlist that admits no metric at all reaches the same empty input: the answer comes + // from the statement, not from whatever the catalog holds. + let query_ctx = native_query_ctx(MetricDiscoveryScope::allowed(std::iter::empty())); + let query = r#"{job="api"}"#; + let plan = plan_remote_native_promql(&engine, &query_ctx, query).await; + assert_regionless_native_plan(&plan, "denied discovery"); + assert_typed_empty_inputs(&plan, "denied discovery"); + let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await; + assert_regionless_physical_plan(&physical, "denied discovery"); + assert_eq!( + 0, + batches_row_count(&batches), + "denied discovery: {physical:?}" + ); + } + /// Plans `sql` and runs the DataFusion analyzer, which is where /// `InsertAssignmentRule` sits. Planning alone stops short of it, so these /// assertions would not see the assignment rewrite at all. diff --git a/src/query/src/promql/planner/native.rs b/src/query/src/promql/planner/native.rs index f9db2da6c64..314178a89b9 100644 --- a/src/query/src/promql/planner/native.rs +++ b/src/query/src/promql/planner/native.rs @@ -1084,6 +1084,13 @@ impl PromPlanner { LogicalPlan::Extension(extension) if extension.node.as_any().is::() ); + // The typed empty frame of a selector that matched no table is already the input: + // it reads no region, so there is no source to wrap it in and no metric to bind to + // it. Wrapping it would name a metric it does not have, which is why it stays + // itself on both routes. + if is_empty_discovered_input(&scan) { + return Ok(scan); + } // A scan stays bare only where a region runs the node over it in process. Over the // wire every input is a raw source, whichever table it reads: the source is what the // transport carries, and it is also what reconciles the fragments of a series that @@ -1483,12 +1490,7 @@ impl PromPlanner { /// frame producer it is instead of looking for a region scan below the node. Its labels are /// explicit, which is the mode a producer names each series in rather than a scan's storage key. fn empty_discovered_input(time_index: String) -> Result { - let schema = Arc::new(SeriesSchema { - fields: Arc::from([FieldSpec { - name: Arc::from(DEFAULT_FIELD_COLUMN), - }]), - eval_grid: None, - }); + let schema = empty_discovered_series_schema(); let frame = raw_series_frame_schema_with_ordering(&schema, &RawLabelSource::Explicit, NATIVE_ORDERING) .context(DataFusionPlanningSnafu)?; @@ -1502,12 +1504,38 @@ fn empty_discovered_input(time_index: String) -> Result { Ok((input, schema, None, time_index, Vec::new(), false)) } +/// The raw series schema of the typed empty input [`empty_discovered_input`] builds. +fn empty_discovered_series_schema() -> Arc { + Arc::new(SeriesSchema { + fields: Arc::from([FieldSpec { + name: Arc::from(DEFAULT_FIELD_COLUMN), + }]), + eval_grid: None, + }) +} + /// Whether `plan` is the typed empty input [`empty_discovered_input`] builds. /// /// It reads no region and holds no scan, so it is not one of the tables a node's binding is derived -/// from: two selectors are "of one table" only if the tables they scan are one. -fn is_empty_discovered_input(plan: &LogicalPlan) -> bool { - matches!(plan, LogicalPlan::EmptyRelation(relation) if !relation.produce_one_row) +/// from: two selectors are "of one table" only if the tables they scan are one. The remote +/// admission gate reads the same shape back, to tell a node with nothing to ship from one that does +/// read regions, so the declared frame is part of the check: an empty relation that does not +/// declare the raw explicit-label frame is some other plan, not this input. +pub(crate) fn is_empty_discovered_input(plan: &LogicalPlan) -> bool { + let LogicalPlan::EmptyRelation(relation) = plan else { + return false; + }; + if relation.produce_one_row { + return false; + } + let Ok(frame) = raw_series_frame_schema_with_ordering( + &empty_discovered_series_schema(), + &RawLabelSource::Explicit, + NATIVE_ORDERING, + ) else { + return false; + }; + relation.schema.as_arrow() == frame.as_ref() } /// The subtrees a [`NativeNode`] reads, in the order the node evaluates them. @@ -2042,6 +2070,10 @@ fn build_evaluated_segment( .samples() .is_some_and(|schema| schema.eval_grid.is_some()) || is_raw_series_source(&child) + // The typed empty frame of a selector that matched no table is an input in its own + // right: it reads no region and carries no metric, so there is nothing to wrap it in. + // A segment over it is still an explicit-label frame with no series. + || is_empty_discovered_input(&child) { inputs.push(child); local_tags.push(source_tags.get(index)?.clone()); diff --git a/tests-integration/src/tests/promql_test.rs b/tests-integration/src/tests/promql_test.rs index 779a172c975..96706d6f80f 100644 --- a/tests-integration/src/tests/promql_test.rs +++ b/tests-integration/src/tests/promql_test.rs @@ -3893,6 +3893,108 @@ async fn distributed_native_promql_remote_refuses_unsupported_discovery_and_name } } +/// The columns and `(timestamp, value)` rows of a statement that reads no table, whose time column +/// is the statement's own rather than a table's. +async fn data_free_rows(output: Output) -> (Vec, Vec<(i64, f64)>) { + let batches = match output.data { + OutputData::Stream(stream) => collect_batches(stream).await.unwrap(), + OutputData::RecordBatches(batches) => batches, + _ => unreachable!("PromQL query returns record batches"), + }; + let mut columns = Vec::new(); + let mut rows = Vec::new(); + for batch in batches { + columns = batch + .schema + .column_schemas() + .iter() + .map(|column| column.name.clone()) + .collect(); + let timestamps = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .columns() + .iter() + .filter(|column| column.data_type() == &DataType::Float64) + .collect::>(); + assert_eq!(values.len(), 1, "one result value column"); + let values = values[0].as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + rows.push((timestamps.value(row), values.value(row))); + } + } + rows.sort_by_key(|row| row.0); + (columns, rows) +} + +/// A statement that reads no table plans natively over the wire, where there is no region to place +/// anything in: the frontend evaluates one fused node, so the answer keeps the native column name +/// instead of the one the existing planner derives from an expression. +#[tokio::test(flavor = "multi_thread")] +async fn distributed_native_promql_data_free_statement_plans_natively() { + common_telemetry::init_default_ut_logging(); + let cluster = GreptimeDbClusterBuilder::new("distributed_native_promql_data_free") + .await + .with_datanodes(2) + .with_real_datanode_grpc_addr(true) + .with_frontend_real_datanode_grpc(true) + .with_query_options(QueryOptions { + experimental_native_promql: true, + ..Default::default() + }) + .build(false) + .await; + let instance = cluster.fe_instance().clone(); + for (expression, expected_columns, expected_rows) in [ + // `start`/`end` are seconds, so this one-step range evaluates at 3000ms. + ("time()", &["time", "value"][..], &[(3000, 3.0)][..]), + ( + r#"label_replace(vector(1), "host", "host1", "", "")"#, + &["time", "value", "host"][..], + &[(3000, 1.0)][..], + ), + ] { + let query = PromQuery { + query: expression.to_string(), + start: "3".to_string(), + end: "3".to_string(), + step: "1s".to_string(), + ..PromQuery::default() + }; + let output = instance + .do_promql_query(&query, native_query_context(true, 2)) + .await + .remove(0) + .unwrap(); + let plan = output.meta.plan.clone().expect("frontend physical plan"); + let shape = native_plan_shape(&plan); + assert_eq!( + 1, shape.fused, + "a data-free statement is one fused node: {plan:?}" + ); + assert!( + shape.regions_per_scan.is_empty(), + "a data-free statement reads no region: {plan:?}" + ); + let (columns, rows) = data_free_rows(output).await; + assert_eq!(expected_columns, columns, "{expression}"); + assert_eq!(expected_rows, rows.as_slice(), "{expression}"); + + // The same statement through the existing planner answers the same rows; only the native + // route names the value column after the tree rather than after an expression. + let legacy = instance + .do_promql_query(&query, native_query_context(false, 2)) + .await + .remove(0) + .unwrap(); + let (_, legacy_rows) = data_free_rows(legacy).await; + assert_eq!(rows, legacy_rows, "{expression} must answer the same rows"); + } +} + #[tokio::test(flavor = "multi_thread")] async fn native_promql_label_rewrites_reach_the_native_route_and_keep_prometheus_answers() { common_telemetry::init_default_ut_logging();