diff --git a/src/promql/src/functions/vector_matching.rs b/src/promql/src/functions/vector_matching.rs index fdef83f6f05..ef96e8aa5ea 100644 --- a/src/promql/src/functions/vector_matching.rs +++ b/src/promql/src/functions/vector_matching.rs @@ -34,6 +34,8 @@ pub enum MatchGroupViolation { ImplicitManyToOne, /// A group modifier left several matches with the same result label set. AmbiguousGroupLabels, + /// A function rewrote labels so that several series share the same label set. + DuplicateLabelSet, } impl MatchGroupViolation { @@ -54,6 +56,9 @@ impl MatchGroupViolation { Self::AmbiguousGroupLabels => format!( "multiple matches for labels {group}: grouping labels must ensure unique matches" ), + Self::DuplicateLabelSet => { + "vector cannot contain metrics with the same labelset".to_string() + } } } } diff --git a/src/query/src/promql/error.rs b/src/query/src/promql/error.rs index 6cd45d9e57c..3b24d33612d 100644 --- a/src/query/src/promql/error.rs +++ b/src/query/src/promql/error.rs @@ -217,12 +217,6 @@ pub enum Error { location: Location, }, - #[snafu(display("vector cannot contain metrics with the same labelset"))] - SameLabelSet { - #[snafu(implicit)] - location: Location, - }, - #[snafu(display("Invalid regular expression in label_replace(): {}", regex))] InvalidRegularExpression { regex: String, @@ -256,7 +250,6 @@ impl ErrorExt for Error { | CombineTableColumnMismatch { .. } | UnexpectedPlanExpr { .. } | UnsupportedMatcherOp { .. } - | SameLabelSet { .. } | TimestampOutOfRange { .. } | SystemTimeOutOfRange { .. } | AtModifierTimestampOutOfRange { .. } diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index dc023762b00..88464bc9ee8 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -107,7 +107,7 @@ use crate::promql::error::{ CatalogSnafu, ColumnNotFoundSnafu, DataFusionPlanningSnafu, ExpectRangeSelectorSnafu, FunctionInvalidArgumentSnafu, InvalidDestinationLabelNameSnafu, InvalidRegularExpressionSnafu, InvalidTimeRangeSnafu, MultiFieldsNotSupportedSnafu, MultipleMetricMatchersSnafu, - MultipleVectorSnafu, NoMetricMatcherSnafu, Result, SameLabelSetSnafu, TableNameNotFoundSnafu, + MultipleVectorSnafu, NoMetricMatcherSnafu, Result, TableNameNotFoundSnafu, TimeIndexNotFoundSnafu, UnexpectedPlanExprSnafu, UnexpectedTokenSnafu, UnknownTableSnafu, UnsupportedExprSnafu, UnsupportedMatcherOpSnafu, ValueNotFoundSnafu, ZeroRangeSelectorSnafu, }; @@ -1752,47 +1752,9 @@ impl PromPlanner { DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone())) }) .collect::>(); - // `timestamp()` preserves the shifted selector timeline even though - // SeriesNormalize now retains raw native timestamp storage. Decimal - // arithmetic shifts before truncating to milliseconds. - let unit_factor = match ident(&time_index_column) - .get_type(normalize.schema()) - .context(DataFusionPlanningSnafu)? - { - ArrowDataType::Timestamp(ArrowTimeUnit::Second, _) => (1_000_i128, 4, 0), - ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, _) => (1, 1, 0), - ArrowDataType::Timestamp(ArrowTimeUnit::Microsecond, _) => (1, 4, 3), - ArrowDataType::Timestamp(ArrowTimeUnit::Nanosecond, _) => (1, 7, 6), - _ => unreachable!("time index is a timestamp"), - }; - let sample_time = ident(&time_index_column) - .cast_to(&ArrowDataType::Int64, normalize.schema()) - .context(DataFusionPlanningSnafu)? - .cast_to(&ArrowDataType::Decimal128(19, 0), normalize.schema()) - .context(DataFusionPlanningSnafu)?; - let sample_time = DfExpr::BinaryExpr(BinaryExpr { - left: Box::new(sample_time), - op: Operator::Multiply, - right: Box::new(lit(ScalarValue::Decimal128( - Some(unit_factor.0), - unit_factor.1, - unit_factor.2, - ))), - }); - let sample_time = DfExpr::BinaryExpr(BinaryExpr { - left: Box::new(sample_time), - op: Operator::Plus, - right: Box::new(lit(ScalarValue::Decimal128(Some(offset_ms as i128), 19, 0))), - }) - .cast_to(&ArrowDataType::Int64, normalize.schema()) - .context(DataFusionPlanningSnafu)? - .cast_to(&ArrowDataType::Float64, normalize.schema()) - .context(DataFusionPlanningSnafu)?; - let sample_time = DfExpr::BinaryExpr(BinaryExpr { - left: Box::new(sample_time), - op: Operator::Divide, - right: Box::new(lit(1000.0)), - }); + // The time index still holds the raw sample timestamp here, which is what + // `timestamp()` reports regardless of `offset` and `@`. + let sample_time = Self::timestamp_seconds_expr(&time_index_column, normalize.schema())?; project_exprs.push(sample_time.alias(×tamp_value_column)); let normalize = LogicalPlanBuilder::from(normalize) .project(project_exprs) @@ -1846,40 +1808,51 @@ impl PromPlanner { }), }; if let Some(timestamp_value_column) = timestamp_value_column { - self.create_timestamp_func_plan(manipulate, ×tamp_value_column) + self.create_timestamp_func_plan(manipulate, ident(timestamp_value_column)) } else { Ok(manipulate) } } - /// Builds a projection plan for the PromQL `timestamp()` function. - /// Projects the time index column as the value column for each row. + /// Converts the timestamp column `column` into PromQL seconds, truncated to milliseconds. + fn timestamp_seconds_expr(column: &str, schema: &DFSchema) -> Result { + let column = DfExpr::Column(Column::from_name(column)); + let ArrowDataType::Timestamp(_, timezone) = + column.get_type(schema).context(DataFusionPlanningSnafu)? + else { + unreachable!("time index is a timestamp") + }; + let millis = column + .cast_to( + &ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, timezone), + schema, + ) + .context(DataFusionPlanningSnafu)? + .cast_to(&ArrowDataType::Int64, schema) + .context(DataFusionPlanningSnafu)? + .cast_to(&ArrowDataType::Float64, schema) + .context(DataFusionPlanningSnafu)?; + Ok(DfExpr::BinaryExpr(BinaryExpr { + left: Box::new(millis), + op: Operator::Divide, + right: Box::new(lit(1000.0)), + })) + } + + /// Builds a projection plan for the PromQL `timestamp()` function, which reports + /// `timestamp_value` as the value of each row, along with the original tag and time index + /// columns. /// - /// # Arguments - /// * `input` - Input [`LogicalPlan`] after instant-vector selection. - /// * `timestamp_value_column` - Private column containing each selected sample's timestamp. - /// - /// # Returns - /// Returns a [`Result`] where the resulting logical plan projects the timestamp - /// column as the value column, along with the original tag and time index columns. - /// - /// # Timestamp vs. Time Function - /// - /// - **Timestamp Function (`timestamp()`)**: In PromQL, the `timestamp()` function returns the - /// timestamp (time index) of each sample as the value column. - /// - /// - **Time Function (`time()`)**: The `time()` function returns the evaluation time of the query - /// as a scalar value. - /// - /// # Side Effects /// Updates the planner context's field columns to the timestamp column name. - /// fn create_timestamp_func_plan( &mut self, input: LogicalPlan, - timestamp_value_column: &str, + timestamp_value: DfExpr, ) -> Result { - let time_expr = ident(timestamp_value_column).alias(DEFAULT_FIELD_COLUMN); + // A row whose fields are all NULL holds no sample, so it must not get a timestamp. The + // check reads the input fields, which the projection below replaces. + let has_sample = self.create_empty_values_filter_expr(true)?; + let time_expr = timestamp_value.alias(DEFAULT_FIELD_COLUMN); self.ctx.field_columns = vec![time_expr.schema_name().to_string()]; let mut project_exprs = Vec::with_capacity(self.ctx.tag_columns.len() + 2); project_exprs.push(self.create_time_index_column_expr()?); @@ -1887,6 +1860,8 @@ impl PromPlanner { project_exprs.extend(self.create_tag_column_exprs()?); LogicalPlanBuilder::from(input) + .filter(has_sample) + .context(DataFusionPlanningSnafu)? .project(project_exprs) .context(DataFusionPlanningSnafu)? .build() @@ -2029,9 +2004,31 @@ impl PromPlanner { // transform function arguments let args = self.create_function_args(&args.args)?; + // Only a vector selector keeps the timestamps of its samples. Any other expression + // produces samples at the evaluation time, which is what `timestamp()` reports for it. + let mut timestamp_arg = args.input.as_ref(); + while let Some(PromExpr::Paren(ParenExpr { expr })) = timestamp_arg { + timestamp_arg = Some(expr); + } + let timestamp_of_selector = + func.name == "timestamp" && matches!(timestamp_arg, Some(PromExpr::VectorSelector(_))); let input = if let Some(prom_expr) = &args.input { - self.prom_expr_to_plan_inner(prom_expr, func.name == "timestamp", query_engine_state) - .await? + let input = self + .prom_expr_to_plan_inner(prom_expr, timestamp_of_selector, query_engine_state) + .await?; + if func.name == "timestamp" && !timestamp_of_selector { + let time_index_column = + self.ctx + .time_index_column + .clone() + .with_context(|| TimeIndexNotFoundSnafu { + table: self.ctx.table_name.clone().unwrap_or_default(), + })?; + let eval_time = Self::timestamp_seconds_expr(&time_index_column, input.schema())?; + self.create_timestamp_func_plan(input, eval_time)? + } else { + input + } } else { self.ctx.time_index_column = Some(SPECIAL_TIME_FUNCTION.to_string()); self.ctx.reset_table_name_and_schema(); @@ -2057,6 +2054,7 @@ impl PromPlanner { // `create_function_expr`: the function that reads it (`predict_linear`) then depends on an // argument instead of on planner state written by the selector below it. let range_fold_offset = self.ctx.range_fold_offset.take(); + let input_tag_columns = self.ctx.tag_columns.clone(); let (mut func_exprs, new_tags) = self.create_function_expr( func, args.literals.clone(), @@ -2108,13 +2106,31 @@ impl PromPlanner { _ => builder, }; + // Rewriting a label the input series already have can map several of them onto the same + // label set, which PromQL rejects. A new label keeps the series distinct. + let may_duplicate_label_sets = matches!(func.name, "label_join" | "label_replace") + && new_tags.iter().any(|tag| input_tag_columns.contains(tag)); + // Update context tags after building plan // We can't push them before planning, because they won't exist until projection. for tag in new_tags { self.ctx.tag_columns.push(tag); } - let plan = builder.build().context(DataFusionPlanningSnafu)?; + let mut plan = builder.build().context(DataFusionPlanningSnafu)?; + if may_duplicate_label_sets { + let labels = self.ctx.tag_columns.clone(); + plan = Self::assert_unique_match_group( + plan, + labels + .iter() + .map(|label| DfExpr::Column(Column::from_name(label))) + .collect(), + labels, + self.create_time_index_column_expr()?, + MatchGroupViolation::DuplicateLabelSet, + )?; + } common_telemetry::debug!("Created PromQL function plan: {plan:?} for {call_expr:?}"); Ok(plan) @@ -3620,6 +3636,7 @@ impl PromPlanner { let (concat_expr, dst_label) = Self::build_concat_labels_expr( &mut other_input_exprs, &self.ctx, + input_schema, query_engine_state, )?; @@ -3641,8 +3658,8 @@ impl PromPlanner { } "label_replace" => { self.ctx.use_tsid = false; - if let Some((replace_expr, dst_label)) = self - .build_regexp_replace_label_expr(&mut other_input_exprs, query_engine_state)? + if let Some((replace_expr, dst_label)) = + self.build_regexp_replace_label_expr(&mut other_input_exprs, input_schema)? { // Reserve the current field columns except the `dst_label`. for value in &self.ctx.field_columns { @@ -3652,10 +3669,8 @@ impl PromPlanner { } } - ensure!( - !self.ctx.tag_columns.contains(&dst_label), - SameLabelSetSnafu - ); + // Remove it from tag columns if exists to avoid duplicated column names + self.ctx.tag_columns.retain(|tag| *tag != dst_label); new_tags.push(dst_label); // Add the new label expr to evaluate exprs.push(replace_expr); @@ -3964,7 +3979,7 @@ impl PromPlanner { fn build_regexp_replace_label_expr( &self, other_input_exprs: &mut VecDeque, - query_engine_state: &QueryEngineState, + input_schema: &DFSchemaRef, ) -> Result> { // label_replace(vector, dst_label, replacement, src_label, regex) let dst_label = match other_input_exprs.pop_front() { @@ -4000,72 +4015,64 @@ impl PromPlanner { .fail()?, }; - // Validate the regex before using it + // Like Prometheus, match the whole source value. A series whose source value matches gets + // `dst_label` set to the expanded replacement; any other series is left unchanged. // doc: https://prometheus.io/docs/prometheus/latest/querying/functions/#label_replace - regex::Regex::new(®ex).map_err(|_| { - InvalidRegularExpressionSnafu { - regex: regex.clone(), - } - .build() - })?; + let anchored = format!("^(?s:{regex})$"); + let compiled = regex::Regex::new(&anchored) + .map_err(|_| InvalidRegularExpressionSnafu { regex }.build())?; + let dst_exists = self.ctx.tag_columns.contains(&dst_label); - // If the src_label exists and regex is empty, keep everything unchanged. - if self.ctx.tag_columns.contains(&src_label) && regex.is_empty() { - return Ok(None); - } - - // If the src_label doesn't exists, and + // A missing source label reads as the empty string for every series, so the result is + // the same for all of them and is decided here. if !self.ctx.tag_columns.contains(&src_label) { - if replacement.is_empty() { - // the replacement is empty, keep everything unchanged. + let Some(captures) = compiled.captures("") else { return Ok(None); - } else { - // the replacement is not empty, always adds dst_label with replacement value. + }; + let mut value = String::new(); + captures.expand(&replacement, &mut value); + if value.is_empty() { + // Setting a label to the empty string removes it, which is a no-op for a new + // label. + if !dst_exists { + return Ok(None); + } return Ok(Some(( - // alias literal `replacement` as dst_label - lit(replacement).alias(&dst_label), + lit(ScalarValue::Utf8(None)).alias(&dst_label), dst_label, ))); } + return Ok(Some((lit(value).alias(&dst_label), dst_label))); } - // Preprocess the regex: - // https://github.com/prometheus/prometheus/blob/d902abc50d6652ba8fe9a81ff8e5cce936114eba/promql/functions.go#L1575C32-L1575C37 - let regex = format!("^(?s:{regex})$"); + let src = Self::label_value_expr(&src_label, input_schema)?; + let matched = DfExpr::ScalarFunction(ScalarFunction { + func: datafusion_functions::regex::regexp_like(), + args: vec![src.clone(), lit(anchored.clone())], + }); + let replaced = Self::empty_label_to_null(DfExpr::ScalarFunction(ScalarFunction { + func: datafusion_functions::regex::regexp_replace(), + args: vec![src, lit(anchored), lit(replacement)], + })); + let unchanged = if dst_exists { + DfExpr::Column(Column::from_name(&dst_label)) + .cast_to(&ArrowDataType::Utf8, input_schema) + .context(DataFusionPlanningSnafu)? + } else { + lit(ScalarValue::Utf8(None)) + }; + let replace_expr = when(matched, replaced) + .otherwise(unchanged) + .context(DataFusionPlanningSnafu)?; - let session_state = query_engine_state.session_state(); - let func = session_state - .scalar_functions() - .get("regexp_replace") - .context(UnsupportedExprSnafu { - name: "regexp_replace", - })?; - - // regexp_replace(src_label, regex, replacement) - let args = vec![ - if src_label.is_empty() { - DfExpr::Literal(ScalarValue::Utf8(Some(String::new())), None) - } else { - DfExpr::Column(Column::from_name(src_label)) - }, - DfExpr::Literal(ScalarValue::Utf8(Some(regex)), None), - DfExpr::Literal(ScalarValue::Utf8(Some(replacement)), None), - ]; - - Ok(Some(( - DfExpr::ScalarFunction(ScalarFunction { - func: func.clone(), - args, - }) - .alias(&dst_label), - dst_label, - ))) + Ok(Some((replace_expr.alias(&dst_label), dst_label))) } /// Build expr for `label_join` function fn build_concat_labels_expr( other_input_exprs: &mut VecDeque, ctx: &PromPlannerContext, + input_schema: &DFSchemaRef, query_engine_state: &QueryEngineState, ) -> Result<(DfExpr, String)> { // label_join(vector, dst_label, separator, src_label_1, src_label_2, ...) @@ -4097,17 +4104,19 @@ impl PromPlanner { let src_labels = other_input_exprs .iter() .map(|expr| { - // Cast source label into column or null literal + // `concat_ws` skips NULL arguments together with their separator, while an + // absent label joins as the empty string. match expr { DfExpr::Literal(ScalarValue::Utf8(Some(label)), None) => { if label.is_empty() { - Ok(DfExpr::Literal(ScalarValue::Null, None)) + FunctionInvalidArgumentSnafu { + fn_name: "label_join", + } + .fail() } else if available_columns.contains(label.as_str()) { - // Label exists in the table schema - Ok(DfExpr::Column(Column::from_name(label))) + Self::label_value_expr(label, input_schema) } else { - // Label doesn't exist, treat as empty string (null) - Ok(DfExpr::Literal(ScalarValue::Null, None)) + Ok(lit("")) } } other => UnexpectedPlanExprSnafu { @@ -4120,12 +4129,10 @@ impl PromPlanner { } }) .collect::>>()?; - ensure!( - !src_labels.is_empty(), - FunctionInvalidArgumentSnafu { - fn_name: "label_join" - } - ); + // Joining no labels yields the empty string, i.e. removes `dst_label`. + if src_labels.is_empty() { + return Ok((lit(ScalarValue::Utf8(None)).alias(&dst_label), dst_label)); + } let session_state = query_engine_state.session_state(); let func = session_state @@ -4139,15 +4146,37 @@ impl PromPlanner { args.extend(src_labels); Ok(( - DfExpr::ScalarFunction(ScalarFunction { + Self::empty_label_to_null(DfExpr::ScalarFunction(ScalarFunction { func: func.clone(), args, - }) + })) .alias(&dst_label), dst_label, )) } + /// The value of `label` as a string, where NULL (the series has no such label) reads as the + /// empty string, as in PromQL. + fn label_value_expr(label: &str, input_schema: &DFSchemaRef) -> Result { + let value = DfExpr::Column(Column::from_name(label)) + .cast_to(&ArrowDataType::Utf8, input_schema) + .context(DataFusionPlanningSnafu)?; + Ok(DfExpr::ScalarFunction(ScalarFunction { + func: coalesce(), + args: vec![value, lit("")], + })) + } + + /// An empty label value means the label is absent in PromQL. Label functions represent it + /// as NULL, like a series that never had the label, so that both compare equal when label + /// sets are matched and neither is reported as a label. + fn empty_label_to_null(value: DfExpr) -> DfExpr { + DfExpr::ScalarFunction(ScalarFunction { + func: datafusion_functions::core::nullif(), + args: vec![value, lit("")], + }) + } + fn create_time_index_column_expr(&self) -> Result { Ok(DfExpr::Column(Column::from_name( self.ctx diff --git a/src/query/src/promql/planner/test.rs b/src/query/src/promql/planner/test.rs index 382f4791e28..215acb85cf9 100644 --- a/src/query/src/promql/planner/test.rs +++ b/src/query/src/promql/planner/test.rs @@ -1699,12 +1699,13 @@ async fn single_timestamp_plan_preserves_source_value() { "Filter: value IS NOT NULL [timestamp:Timestamp(ms), value:Float64, tag_0:Utf8]\ \n Projection: some_metric.timestamp, value AS value, some_metric.tag_0 [timestamp:Timestamp(ms), value:Float64, tag_0:Utf8]\ \n Projection: some_metric.timestamp, __promql_timestamp_value_ AS value, some_metric.tag_0 [timestamp:Timestamp(ms), value:Float64, tag_0:Utf8]\ - \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ - \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, CAST(CAST(CAST(CAST(some_metric.timestamp AS Int64) AS Decimal128(19, 0)) * Decimal128(1,1,0) + Decimal128(0,19,0) AS Int64) AS Float64) / Float64(1000) AS __promql_timestamp_value_ [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ - \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ - \n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ - \n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ - \n TableScan: some_metric [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]", + \n Filter: some_metric.field_0 IS NOT NULL [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ + \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ + \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, CAST(CAST(some_metric.timestamp AS Int64) AS Float64) / Float64(1000) AS __promql_timestamp_value_ [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ + \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n TableScan: some_metric [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]", ); assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -2413,9 +2414,9 @@ async fn at_modifier_keeps_multi_series_roots_out_of_promoted_subtree() { async fn at_modifier_does_not_promote_label_join() { for query in [ // Directly above the anchored instant selector... - "label_join(some_metric @ 300, \"tag_0\", \"-\", \"\")", + "label_join(some_metric @ 300, \"tag_0\", \"-\", \"tag_0\", \"tag_0\")", // ... and below another call, which is planned as usual over the join. - "abs(label_join(some_metric @ 300, \"tag_0\", \"-\", \"\"))", + "abs(label_join(some_metric @ 300, \"tag_0\", \"-\", \"tag_0\", \"tag_0\"))", ] { let plan = build_at_modifier_plan(query, 0, 1000).await; let plan_str = plan.display_indent_schema().to_string(); @@ -2448,7 +2449,7 @@ async fn at_modifier_does_not_promote_label_join() { // A range call below the join is still promoted on its own: the anchored window is folded // once per series, and the join above it is evaluated at every step over that replay. let plan = build_at_modifier_plan( - "label_join(rate(some_metric[5m] @ 300), \"tag_0\", \"-\", \"\")", + "label_join(rate(some_metric[5m] @ 300), \"tag_0\", \"-\", \"tag_0\", \"tag_0\")", 0, 1000, ) @@ -5902,7 +5903,7 @@ async fn test_label_join() { let expected = r#" Filter: up.field_0 IS NOT NULL [timestamp:Timestamp(ms), field_0:Float64;N, foo:Utf8;N, tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8] - Projection: up.timestamp, up.field_0, concat_ws(Utf8(","), up.tag_1, up.tag_2, up.tag_3) AS foo, up.tag_0, up.tag_1, up.tag_2, up.tag_3 [timestamp:Timestamp(ms), field_0:Float64;N, foo:Utf8;N, tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8] + Projection: up.timestamp, up.field_0, nullif(concat_ws(Utf8(","), coalesce(up.tag_1, Utf8("")), coalesce(up.tag_2, Utf8("")), coalesce(up.tag_3, Utf8(""))), Utf8("")) AS foo, up.tag_0, up.tag_1, up.tag_2, up.tag_3 [timestamp:Timestamp(ms), field_0:Float64;N, foo:Utf8;N, tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8] PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, timestamp:Timestamp(ms), field_0:Float64;N] PromSeriesDivide: tags=["tag_0", "tag_1", "tag_2", "tag_3"] [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, timestamp:Timestamp(ms), field_0:Float64;N] Sort: up.tag_0 ASC NULLS FIRST, up.tag_1 ASC NULLS FIRST, up.tag_2 ASC NULLS FIRST, up.tag_3 ASC NULLS FIRST, up.timestamp ASC NULLS FIRST [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, timestamp:Timestamp(ms), field_0:Float64;N] @@ -5937,7 +5938,7 @@ async fn test_label_replace() { let expected = r#" Filter: up.field_0 IS NOT NULL [timestamp:Timestamp(ms), field_0:Float64;N, foo:Utf8;N, tag_0:Utf8] - Projection: up.timestamp, up.field_0, regexp_replace(up.tag_0, Utf8("^(?s:(.*):.*)$"), Utf8("$1")) AS foo, up.tag_0 [timestamp:Timestamp(ms), field_0:Float64;N, foo:Utf8;N, tag_0:Utf8] + Projection: up.timestamp, up.field_0, CASE WHEN regexp_like(coalesce(up.tag_0, Utf8("")), Utf8("^(?s:(.*):.*)$")) THEN nullif(regexp_replace(coalesce(up.tag_0, Utf8("")), Utf8("^(?s:(.*):.*)$"), Utf8("$1")), Utf8("")) ELSE Utf8(NULL) END AS foo, up.tag_0 [timestamp:Timestamp(ms), field_0:Float64;N, foo:Utf8;N, tag_0:Utf8] PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N] PromSeriesDivide: tags=["tag_0"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N] Sort: up.tag_0 ASC NULLS FIRST, up.timestamp ASC NULLS FIRST [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N] diff --git a/src/query/src/promql/planner/test/delta.rs b/src/query/src/promql/planner/test/delta.rs index a647821e0b3..f0c918351d1 100644 --- a/src/query/src/promql/planner/test/delta.rs +++ b/src/query/src/promql/planner/test/delta.rs @@ -1102,12 +1102,12 @@ async fn delta_offsets_survive_optimized_plan_serialization() { ( "timestamp positive offset", r#"timestamp(delta_metric{series="cumulative"} offset 60s)"#, - 120.0, + 60.0, ), ( "timestamp negative offset", r#"timestamp(delta_metric{series="cumulative"} offset -60s)"#, - 120.0, + 180.0, ), ( "range positive offset", diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index bd9226f724b..22166560437 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1022,6 +1022,31 @@ pub async fn test_prometheus_label_replace_response(store_type: StorageType) { .unwrap() ); + // A series whose source value does not match keeps its labels, without `host_copy`. + let query = encode(r#"label_replace(demo, "host_copy", "$1", "host", "other(.*)")"#); + let res = client + .get(&format!("/v1/prometheus/api/v1/query?query={query}&time=0")) + .send() + .await; + + assert_eq!(res.status(), StatusCode::OK); + let body = res.json::().await; + assert_eq!(body.status, "success"); + assert_eq!( + body.data, + serde_json::from_value::(json!({ + "resultType": "vector", + "result": [{ + "metric": { + "__name__": "demo", + "host": "host1" + }, + "value": [0.0, "1.1"] + }] + })) + .unwrap() + ); + guard.remove_all().await; } diff --git a/tests/cases/standalone/common/promql/at_modifier.result b/tests/cases/standalone/common/promql/at_modifier.result index 0a19eac692c..27570231833 100644 --- a/tests/cases/standalone/common/promql/at_modifier.result +++ b/tests/cases/standalone/common/promql/at_modifier.result @@ -621,9 +621,6 @@ TQL EVAL (330, 390, '60s') predict_linear(at_modifier_gauge{host="a"}[5m] offset -- on its own and `timestamp()` reports the timestamp of the sample the anchor selected at every -- step (300s here). Its plan keeps the anchored selection below the replay of the grid and projects -- the timestamp above it. --- --- Only an `@` without an `offset` is asserted here: the value of `timestamp()` for an anchored --- selector that also carries an `offset` is a pre-existing question of its own, out of scope here. -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 240, '60s') timestamp(at_modifier_gauge @ 300); @@ -642,95 +639,105 @@ TQL EVAL (0, 240, '60s') timestamp(at_modifier_gauge @ 300); | 1970-01-01T00:04:00 | 300.0 | b | +---------------------+-------+------+ +-- With an `offset`, the anchor selects the sample at 240s, and `timestamp()` reports 240s. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 120, '60s') timestamp(at_modifier_gauge @ 300 offset 1m); + ++---------------------+-------+------+ +| ts | value | host | ++---------------------+-------+------+ +| 1970-01-01T00:00:00 | 240.0 | a | +| 1970-01-01T00:00:00 | 240.0 | b | +| 1970-01-01T00:01:00 | 240.0 | a | +| 1970-01-01T00:01:00 | 240.0 | b | +| 1970-01-01T00:02:00 | 240.0 | a | +| 1970-01-01T00:02:00 | 240.0 | b | ++---------------------+-------+------+ + -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN (0, 240, '60s') timestamp(at_modifier_gauge @ 300); -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | -| | Projection: at_modifier_gauge.ts, value AS value, at_modifier_gauge.host | -| | Projection: at_modifier_gauge.ts, __promql_timestamp_value_ AS value, at_modifier_gauge.host | -| | PromInstantManipulate: range=[0..240000], lookback=[240001], interval=[60000], time index=[ts] | -| | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[60000], time index=[ts] | -| | Projection: at_modifier_gauge.ts, at_modifier_gauge.val, at_modifier_gauge.host, CAST(CAST(CAST(CAST(CAST(at_modifier_gauge.ts AS Int64) AS Decimal128(19, 0)) AS Decimal128(21, 0)) + Decimal128(0,19,0) AS Int64) AS Float64) / Float64(1000) AS __promql_timestamp_value_ | -| | PromSeriesNormalize: offset=[-300000], time index=[ts], filter NaN: [false] | -| | PromSeriesDivide: tags=["host"] | -| | Sort: at_modifier_gauge.host ASC NULLS FIRST, at_modifier_gauge.ts ASC NULLS FIRST | -| | Filter: at_modifier_gauge.ts >= TimestampMillisecond(1, None) AND at_modifier_gauge.ts <= TimestampMillisecond(300000, None) | -| | TableScan: at_modifier_gauge, partial_filters=[at_modifier_gauge.ts >= TimestampMillisecond(1, None), at_modifier_gauge.ts <= TimestampMillisecond(300000, None)] | -| | ]] | -| physical_plan | CooperativeExec | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: at_modifier_gauge.ts, value AS value, at_modifier_gauge.host | +| | Projection: at_modifier_gauge.ts, __promql_timestamp_value_ AS value, at_modifier_gauge.host | +| | Filter: at_modifier_gauge.val IS NOT NULL | +| | PromInstantManipulate: range=[0..240000], lookback=[240001], interval=[60000], time index=[ts] | +| | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[60000], time index=[ts] | +| | Projection: at_modifier_gauge.ts, at_modifier_gauge.val, at_modifier_gauge.host, CAST(CAST(at_modifier_gauge.ts AS Int64) AS Float64) / Float64(1000) AS __promql_timestamp_value_ | +| | PromSeriesNormalize: offset=[-300000], time index=[ts], filter NaN: [false] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: at_modifier_gauge.host ASC NULLS FIRST, at_modifier_gauge.ts ASC NULLS FIRST | +| | Filter: at_modifier_gauge.ts >= TimestampMillisecond(1, None) AND at_modifier_gauge.ts <= TimestampMillisecond(300000, None) | +| | TableScan: at_modifier_gauge, partial_filters=[at_modifier_gauge.ts >= TimestampMillisecond(1, None), at_modifier_gauge.ts <= TimestampMillisecond(300000, None)] | +| | ]] | +| physical_plan | CooperativeExec | | | MergeScanExec: REDACTED -| | | -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- 11. `label_join` above an anchored selector. The call rewrites the label the input series are --- told apart by (`host` becomes the empty string), so it is never promoted: it must stay above the +-- told apart by (`host` becomes `a-a` and `b-b`), so it is never promoted: it must stay above the -- per-series replay of the selector and be evaluated at every step. Both hosts are still reported, -- with their own value, at every step. A promoted `label_join` would instead be replayed through -- the labels it just rewrote and report the two series as a single timeline, i.e. one mixed row -- per step. --- --- The invariant asserted here is scoped to the replay: it must not drop or mix the input rows. --- It is not a claim about the final PromQL semantics of this query: joining `host` to one value --- leaves two samples with the same label set at the same timestamp, which Prometheus rejects, --- while `label_join` does not validate that yet (`label_replace` errors on such a rewrite --- instead). That duplicate-labelset validation gap is pre-existing and out of scope here. -- SQLNESS SORT_RESULT 3 1 -TQL EVAL (300, 480, '60s') label_join(at_modifier_gauge @ 300, "host", "", ""); +TQL EVAL (300, 480, '60s') label_join(at_modifier_gauge @ 300, "host", "-", "host", "host"); +---------------------+------+------+ | ts | val | host | +---------------------+------+------+ -| 1970-01-01T00:05:00 | 12.0 | | -| 1970-01-01T00:05:00 | 5.0 | | -| 1970-01-01T00:06:00 | 12.0 | | -| 1970-01-01T00:06:00 | 5.0 | | -| 1970-01-01T00:07:00 | 12.0 | | -| 1970-01-01T00:07:00 | 5.0 | | -| 1970-01-01T00:08:00 | 12.0 | | -| 1970-01-01T00:08:00 | 5.0 | | +| 1970-01-01T00:05:00 | 12.0 | b-b | +| 1970-01-01T00:05:00 | 5.0 | a-a | +| 1970-01-01T00:06:00 | 12.0 | b-b | +| 1970-01-01T00:06:00 | 5.0 | a-a | +| 1970-01-01T00:07:00 | 12.0 | b-b | +| 1970-01-01T00:07:00 | 5.0 | a-a | +| 1970-01-01T00:08:00 | 12.0 | b-b | +| 1970-01-01T00:08:00 | 5.0 | a-a | +---------------------+------+------+ -- The same join below another call: neither is promoted, and both are evaluated at every step over -- the per-series replay of the anchored selector. -- SQLNESS SORT_RESULT 3 1 -TQL EVAL (300, 480, '60s') abs(label_join(at_modifier_gauge @ 300, "host", "", "")); +TQL EVAL (300, 480, '60s') abs(label_join(at_modifier_gauge @ 300, "host", "-", "host", "host")); +---------------------+----------+------+ | ts | abs(val) | host | +---------------------+----------+------+ -| 1970-01-01T00:05:00 | 12.0 | | -| 1970-01-01T00:05:00 | 5.0 | | -| 1970-01-01T00:06:00 | 12.0 | | -| 1970-01-01T00:06:00 | 5.0 | | -| 1970-01-01T00:07:00 | 12.0 | | -| 1970-01-01T00:07:00 | 5.0 | | -| 1970-01-01T00:08:00 | 12.0 | | -| 1970-01-01T00:08:00 | 5.0 | | +| 1970-01-01T00:05:00 | 12.0 | b-b | +| 1970-01-01T00:05:00 | 5.0 | a-a | +| 1970-01-01T00:06:00 | 12.0 | b-b | +| 1970-01-01T00:06:00 | 5.0 | a-a | +| 1970-01-01T00:07:00 | 12.0 | b-b | +| 1970-01-01T00:07:00 | 5.0 | a-a | +| 1970-01-01T00:08:00 | 12.0 | b-b | +| 1970-01-01T00:08:00 | 5.0 | a-a | +---------------------+----------+------+ -- A range call below the join is still promoted on its own (it is the direct call over the -- anchored range selector): the anchored window is folded once per series, and the join above that -- replay reports the rate of both hosts at every step. -- SQLNESS SORT_RESULT 3 1 -TQL EVAL (300, 480, '60s') label_join(rate(at_modifier_counter_total[5m] @ 300), "host", "", ""); +TQL EVAL (300, 480, '60s') label_join(rate(at_modifier_counter_total[5m] @ 300), "host", "-", "host", "host"); +---------------------+------------------------------------------+------+ | ts | prom_rate(ts_range,val,ts,Int64(300000)) | host | +---------------------+------------------------------------------+------+ -| 1970-01-01T00:05:00 | 0.5 | | -| 1970-01-01T00:05:00 | 1.0 | | -| 1970-01-01T00:06:00 | 0.5 | | -| 1970-01-01T00:06:00 | 1.0 | | -| 1970-01-01T00:07:00 | 0.5 | | -| 1970-01-01T00:07:00 | 1.0 | | -| 1970-01-01T00:08:00 | 0.5 | | -| 1970-01-01T00:08:00 | 1.0 | | +| 1970-01-01T00:05:00 | 0.5 | b-b | +| 1970-01-01T00:05:00 | 1.0 | a-a | +| 1970-01-01T00:06:00 | 0.5 | b-b | +| 1970-01-01T00:06:00 | 1.0 | a-a | +| 1970-01-01T00:07:00 | 0.5 | b-b | +| 1970-01-01T00:07:00 | 1.0 | a-a | +| 1970-01-01T00:08:00 | 0.5 | b-b | +| 1970-01-01T00:08:00 | 1.0 | a-a | +---------------------+------------------------------------------+------+ DROP TABLE at_modifier_gauge; diff --git a/tests/cases/standalone/common/promql/at_modifier.sql b/tests/cases/standalone/common/promql/at_modifier.sql index 9e9bc21a9c1..b4fd9d1a552 100644 --- a/tests/cases/standalone/common/promql/at_modifier.sql +++ b/tests/cases/standalone/common/promql/at_modifier.sql @@ -258,12 +258,13 @@ TQL EVAL (330, 390, '60s') predict_linear(at_modifier_gauge{host="a"}[5m] offset -- on its own and `timestamp()` reports the timestamp of the sample the anchor selected at every -- step (300s here). Its plan keeps the anchored selection below the replay of the grid and projects -- the timestamp above it. --- --- Only an `@` without an `offset` is asserted here: the value of `timestamp()` for an anchored --- selector that also carries an `offset` is a pre-existing question of its own, out of scope here. -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 240, '60s') timestamp(at_modifier_gauge @ 300); +-- With an `offset`, the anchor selects the sample at 240s, and `timestamp()` reports 240s. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 120, '60s') timestamp(at_modifier_gauge @ 300 offset 1m); + -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -271,30 +272,24 @@ TQL EVAL (0, 240, '60s') timestamp(at_modifier_gauge @ 300); TQL EXPLAIN (0, 240, '60s') timestamp(at_modifier_gauge @ 300); -- 11. `label_join` above an anchored selector. The call rewrites the label the input series are --- told apart by (`host` becomes the empty string), so it is never promoted: it must stay above the +-- told apart by (`host` becomes `a-a` and `b-b`), so it is never promoted: it must stay above the -- per-series replay of the selector and be evaluated at every step. Both hosts are still reported, -- with their own value, at every step. A promoted `label_join` would instead be replayed through -- the labels it just rewrote and report the two series as a single timeline, i.e. one mixed row -- per step. --- --- The invariant asserted here is scoped to the replay: it must not drop or mix the input rows. --- It is not a claim about the final PromQL semantics of this query: joining `host` to one value --- leaves two samples with the same label set at the same timestamp, which Prometheus rejects, --- while `label_join` does not validate that yet (`label_replace` errors on such a rewrite --- instead). That duplicate-labelset validation gap is pre-existing and out of scope here. -- SQLNESS SORT_RESULT 3 1 -TQL EVAL (300, 480, '60s') label_join(at_modifier_gauge @ 300, "host", "", ""); +TQL EVAL (300, 480, '60s') label_join(at_modifier_gauge @ 300, "host", "-", "host", "host"); -- The same join below another call: neither is promoted, and both are evaluated at every step over -- the per-series replay of the anchored selector. -- SQLNESS SORT_RESULT 3 1 -TQL EVAL (300, 480, '60s') abs(label_join(at_modifier_gauge @ 300, "host", "", "")); +TQL EVAL (300, 480, '60s') abs(label_join(at_modifier_gauge @ 300, "host", "-", "host", "host")); -- A range call below the join is still promoted on its own (it is the direct call over the -- anchored range selector): the anchored window is folded once per series, and the join above that -- replay reports the rate of both hosts at every step. -- SQLNESS SORT_RESULT 3 1 -TQL EVAL (300, 480, '60s') label_join(rate(at_modifier_counter_total[5m] @ 300), "host", "", ""); +TQL EVAL (300, 480, '60s') label_join(rate(at_modifier_counter_total[5m] @ 300), "host", "-", "host", "host"); DROP TABLE at_modifier_gauge; diff --git a/tests/cases/standalone/common/promql/label.result b/tests/cases/standalone/common/promql/label.result index 112b6261715..3fe8d559221 100644 --- a/tests/cases/standalone/common/promql/label.result +++ b/tests/cases/standalone/common/promql/label.result @@ -21,9 +21,23 @@ INSERT INTO TABLE test VALUES Affected Rows: 8 -- Missing source labels -- +-- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "new_host", "-"); -Error: 1004(InvalidArguments), Invalid function argument for label_join ++---------------------+-----+----------+-------+------------+ +| ts | val | new_host | host | idc | ++---------------------+-----+----------+-------+------------+ +| 1970-01-01T00:00:00 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 3 | | host1 | idc2:zone1 | +| 1970-01-01T00:00:10 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:10 | 3 | | host1 | idc2:zone1 | +| 1970-01-01T00:00:10 | 5 | | host1 | idc3:zone2 | +| 1970-01-01T00:00:15 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:15 | 3 | | host1 | idc2:zone1 | +| 1970-01-01T00:00:15 | 5 | | host1 | idc3:zone2 | +| 1970-01-01T00:00:15 | 7 | | host1 | idc4:zone3 | ++---------------------+-----+----------+-------+------------+ -- dst_label is equal to source label -- -- SQLNESS SORT_RESULT 3 1 @@ -63,24 +77,27 @@ TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "host", "-", "idc", "host" | 1970-01-01T00:00:15 | 7 | idc4:zone3-host1 | idc4:zone3 | +---------------------+-----+------------------+------------+ --- test the empty source label -- --- SQLNESS SORT_RESULT 3 1 +-- Absent labels join as empty strings and keep their separators -- +TQL EVAL (0, 15, '5s') label_join(label_join(vector(1), "a", "", "missing"), "b", "-", "a", "a"); + ++---------------------+----------------+---+---+ +| time | greptime_value | b | a | ++---------------------+----------------+---+---+ +| 1970-01-01T00:00:00 | 1.0 | - | | +| 1970-01-01T00:00:05 | 1.0 | - | | +| 1970-01-01T00:00:10 | 1.0 | - | | +| 1970-01-01T00:00:15 | 1.0 | - | | ++---------------------+----------------+---+---+ + +-- An empty source label name is invalid -- TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "host", "-", ""); -+---------------------+-----+------+------------+ -| ts | val | host | idc | -+---------------------+-----+------+------------+ -| 1970-01-01T00:00:00 | 1 | | idc1 | -| 1970-01-01T00:00:05 | 1 | | idc1 | -| 1970-01-01T00:00:05 | 3 | | idc2:zone1 | -| 1970-01-01T00:00:10 | 1 | | idc1 | -| 1970-01-01T00:00:10 | 3 | | idc2:zone1 | -| 1970-01-01T00:00:10 | 5 | | idc3:zone2 | -| 1970-01-01T00:00:15 | 1 | | idc1 | -| 1970-01-01T00:00:15 | 3 | | idc2:zone1 | -| 1970-01-01T00:00:15 | 5 | | idc3:zone2 | -| 1970-01-01T00:00:15 | 7 | | idc4:zone3 | -+---------------------+-----+------+------------+ +Error: 1004(InvalidArguments), Invalid function argument for label_join + +-- Both hosts are joined to the same `host` at 0s, leaving two series with the same label set -- +TQL EVAL (0, 15, '5s') label_join(test, "host", "-", "idc"); + +Error: 3001(EngineExecuteQuery), Execution error: vector cannot contain metrics with the same labelset -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "new_host", "-", "idc", "host"); @@ -114,13 +131,13 @@ TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "new_idc", "$2", "idc", +---------------------+-----+---------+-------+------------+ | ts | val | new_idc | host | idc | +---------------------+-----+---------+-------+------------+ -| 1970-01-01T00:00:00 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:05 | 1 | idc1 | host1 | idc1 | +| 1970-01-01T00:00:00 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 1 | | host1 | idc1 | | 1970-01-01T00:00:05 | 3 | zone1 | host1 | idc2:zone1 | -| 1970-01-01T00:00:10 | 1 | idc1 | host1 | idc1 | +| 1970-01-01T00:00:10 | 1 | | host1 | idc1 | | 1970-01-01T00:00:10 | 3 | zone1 | host1 | idc2:zone1 | | 1970-01-01T00:00:10 | 5 | zone2 | host1 | idc3:zone2 | -| 1970-01-01T00:00:15 | 1 | idc1 | host1 | idc1 | +| 1970-01-01T00:00:15 | 1 | | host1 | idc1 | | 1970-01-01T00:00:15 | 3 | zone1 | host1 | idc2:zone1 | | 1970-01-01T00:00:15 | 5 | zone2 | host1 | idc3:zone2 | | 1970-01-01T00:00:15 | 7 | zone3 | host1 | idc4:zone3 | @@ -129,20 +146,20 @@ TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "new_idc", "$2", "idc", -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "new_idc", "idc99", "idc", "idc2.*"); -+---------------------+-----+------------+-------+------------+ -| ts | val | new_idc | host | idc | -+---------------------+-----+------------+-------+------------+ -| 1970-01-01T00:00:00 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:05 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:05 | 3 | idc99 | host1 | idc2:zone1 | -| 1970-01-01T00:00:10 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:10 | 3 | idc99 | host1 | idc2:zone1 | -| 1970-01-01T00:00:10 | 5 | idc3:zone2 | host1 | idc3:zone2 | -| 1970-01-01T00:00:15 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:15 | 3 | idc99 | host1 | idc2:zone1 | -| 1970-01-01T00:00:15 | 5 | idc3:zone2 | host1 | idc3:zone2 | -| 1970-01-01T00:00:15 | 7 | idc4:zone3 | host1 | idc4:zone3 | -+---------------------+-----+------------+-------+------------+ ++---------------------+-----+---------+-------+------------+ +| ts | val | new_idc | host | idc | ++---------------------+-----+---------+-------+------------+ +| 1970-01-01T00:00:00 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 3 | idc99 | host1 | idc2:zone1 | +| 1970-01-01T00:00:10 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:10 | 3 | idc99 | host1 | idc2:zone1 | +| 1970-01-01T00:00:10 | 5 | | host1 | idc3:zone2 | +| 1970-01-01T00:00:15 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:15 | 3 | idc99 | host1 | idc2:zone1 | +| 1970-01-01T00:00:15 | 5 | | host1 | idc3:zone2 | +| 1970-01-01T00:00:15 | 7 | | host1 | idc4:zone3 | ++---------------------+-----+---------+-------+------------+ -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "new_idc", "$2", "idc", "(.*):(.*)"); @@ -150,26 +167,61 @@ TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "new_idc", "$2", "idc", +---------------------+-----+---------+-------+------+ | ts | val | new_idc | host | idc | +---------------------+-----+---------+-------+------+ -| 1970-01-01T00:00:00 | 2 | idc1 | host2 | idc1 | -| 1970-01-01T00:00:05 | 2 | idc1 | host2 | idc1 | -| 1970-01-01T00:00:05 | 4 | idc2 | host2 | idc2 | -| 1970-01-01T00:00:10 | 2 | idc1 | host2 | idc1 | -| 1970-01-01T00:00:10 | 4 | idc2 | host2 | idc2 | -| 1970-01-01T00:00:10 | 6 | idc3 | host2 | idc3 | -| 1970-01-01T00:00:15 | 2 | idc1 | host2 | idc1 | -| 1970-01-01T00:00:15 | 4 | idc2 | host2 | idc2 | -| 1970-01-01T00:00:15 | 6 | idc3 | host2 | idc3 | -| 1970-01-01T00:00:15 | 8 | idc4 | host2 | idc4 | +| 1970-01-01T00:00:00 | 2 | | host2 | idc1 | +| 1970-01-01T00:00:05 | 2 | | host2 | idc1 | +| 1970-01-01T00:00:05 | 4 | | host2 | idc2 | +| 1970-01-01T00:00:10 | 2 | | host2 | idc1 | +| 1970-01-01T00:00:10 | 4 | | host2 | idc2 | +| 1970-01-01T00:00:10 | 6 | | host2 | idc3 | +| 1970-01-01T00:00:15 | 2 | | host2 | idc1 | +| 1970-01-01T00:00:15 | 4 | | host2 | idc2 | +| 1970-01-01T00:00:15 | 6 | | host2 | idc3 | +| 1970-01-01T00:00:15 | 8 | | host2 | idc4 | +---------------------+-----+---------+-------+------+ -- dst_label is equal to source label -- -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "idc", "$2", "idc", "(.*):(.*)"); -Error: 1004(InvalidArguments), vector cannot contain metrics with the same labelset ++---------------------+-----+------+-------+ +| ts | val | idc | host | ++---------------------+-----+------+-------+ +| 1970-01-01T00:00:00 | 2 | idc1 | host2 | +| 1970-01-01T00:00:05 | 2 | idc1 | host2 | +| 1970-01-01T00:00:05 | 4 | idc2 | host2 | +| 1970-01-01T00:00:10 | 2 | idc1 | host2 | +| 1970-01-01T00:00:10 | 4 | idc2 | host2 | +| 1970-01-01T00:00:10 | 6 | idc3 | host2 | +| 1970-01-01T00:00:15 | 2 | idc1 | host2 | +| 1970-01-01T00:00:15 | 4 | idc2 | host2 | +| 1970-01-01T00:00:15 | 6 | idc3 | host2 | +| 1970-01-01T00:00:15 | 8 | idc4 | host2 | ++---------------------+-----+------+-------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "idc", "$2", "idc", "(.*):(.*)"); + ++---------------------+-----+-------+-------+ +| ts | val | idc | host | ++---------------------+-----+-------+-------+ +| 1970-01-01T00:00:00 | 1 | idc1 | host1 | +| 1970-01-01T00:00:05 | 1 | idc1 | host1 | +| 1970-01-01T00:00:05 | 3 | zone1 | host1 | +| 1970-01-01T00:00:10 | 1 | idc1 | host1 | +| 1970-01-01T00:00:10 | 3 | zone1 | host1 | +| 1970-01-01T00:00:10 | 5 | zone2 | host1 | +| 1970-01-01T00:00:15 | 1 | idc1 | host1 | +| 1970-01-01T00:00:15 | 3 | zone1 | host1 | +| 1970-01-01T00:00:15 | 5 | zone2 | host1 | +| 1970-01-01T00:00:15 | 7 | zone3 | host1 | ++---------------------+-----+-------+-------+ + +-- Both hosts are rewritten to the same `host`, leaving two series with the same label set -- +TQL EVAL (0, 15, '5s') label_replace(test, "host", "x", "host", ".*"); + +Error: 3001(EngineExecuteQuery), Execution error: vector cannot contain metrics with the same labelset -- test the empty source label -- --- TODO(dennis): we can't remove the label currently -- -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "idc2", "", "", ""); @@ -221,20 +273,20 @@ TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} + label_replace(vector(1), -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') label_replace(test{host="host1"}, "host2", "host2", "host", ""); -+---------------------+-----+-------+------------+ -| ts | val | host | idc | -+---------------------+-----+-------+------------+ -| 1970-01-01T00:00:00 | 1 | host1 | idc1 | -| 1970-01-01T00:00:05 | 1 | host1 | idc1 | -| 1970-01-01T00:00:05 | 3 | host1 | idc2:zone1 | -| 1970-01-01T00:00:10 | 1 | host1 | idc1 | -| 1970-01-01T00:00:10 | 3 | host1 | idc2:zone1 | -| 1970-01-01T00:00:10 | 5 | host1 | idc3:zone2 | -| 1970-01-01T00:00:15 | 1 | host1 | idc1 | -| 1970-01-01T00:00:15 | 3 | host1 | idc2:zone1 | -| 1970-01-01T00:00:15 | 5 | host1 | idc3:zone2 | -| 1970-01-01T00:00:15 | 7 | host1 | idc4:zone3 | -+---------------------+-----+-------+------------+ ++---------------------+-----+-------+-------+------------+ +| ts | val | host2 | host | idc | ++---------------------+-----+-------+-------+------------+ +| 1970-01-01T00:00:00 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 3 | | host1 | idc2:zone1 | +| 1970-01-01T00:00:10 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:10 | 3 | | host1 | idc2:zone1 | +| 1970-01-01T00:00:10 | 5 | | host1 | idc3:zone2 | +| 1970-01-01T00:00:15 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:15 | 3 | | host1 | idc2:zone1 | +| 1970-01-01T00:00:15 | 5 | | host1 | idc3:zone2 | +| 1970-01-01T00:00:15 | 7 | | host1 | idc4:zone3 | ++---------------------+-----+-------+-------+------------+ -- Empty regex with not existing source label -- SQLNESS SORT_RESULT 3 1 @@ -274,6 +326,23 @@ TQL EVAL(0, 15, '5s') label_replace(test{host="host1"}, "host2", "", "instance", | 1970-01-01T00:00:15 | 7 | host1 | idc4:zone3 | +---------------------+-----+-------+------------+ +-- An empty replacement removes an existing label -- +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') label_replace(test{idc="idc1"}, "idc", "", "instance", ""); + ++---------------------+-----+-----+-------+ +| ts | val | idc | host | ++---------------------+-----+-----+-------+ +| 1970-01-01T00:00:00 | 1 | | host1 | +| 1970-01-01T00:00:00 | 2 | | host2 | +| 1970-01-01T00:00:05 | 1 | | host1 | +| 1970-01-01T00:00:05 | 2 | | host2 | +| 1970-01-01T00:00:10 | 1 | | host1 | +| 1970-01-01T00:00:10 | 2 | | host2 | +| 1970-01-01T00:00:15 | 1 | | host1 | +| 1970-01-01T00:00:15 | 2 | | host2 | ++---------------------+-----+-----+-------+ + -- Empty regex and different label value -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} * label_replace(vector(1), "host", "host2", "host", ""); @@ -307,10 +376,10 @@ TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "new_idc", "idc99", "id +---------------------+-----+---------+-------+------+ | ts | val | new_idc | host | idc | +---------------------+-----+---------+-------+------+ -| 1970-01-01T00:00:00 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:05 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:10 | 1 | idc1 | host1 | idc1 | -| 1970-01-01T00:00:15 | 1 | idc1 | host1 | idc1 | +| 1970-01-01T00:00:00 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:05 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:10 | 1 | | host1 | idc1 | +| 1970-01-01T00:00:15 | 1 | | host1 | idc1 | +---------------------+-----+---------+-------+------+ -- SQLNESS SORT_RESULT 3 1 diff --git a/tests/cases/standalone/common/promql/label.sql b/tests/cases/standalone/common/promql/label.sql index 6a77e9b5214..6c2d64d42ac 100644 --- a/tests/cases/standalone/common/promql/label.sql +++ b/tests/cases/standalone/common/promql/label.sql @@ -17,6 +17,7 @@ INSERT INTO TABLE test VALUES (15000, 'host2', 'idc4',8); -- Missing source labels -- +-- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "new_host", "-"); -- dst_label is equal to source label -- @@ -27,10 +28,15 @@ TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "host", "-", "host"); -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "host", "-", "idc", "host"); --- test the empty source label -- --- SQLNESS SORT_RESULT 3 1 +-- Absent labels join as empty strings and keep their separators -- +TQL EVAL (0, 15, '5s') label_join(label_join(vector(1), "a", "", "missing"), "b", "-", "a", "a"); + +-- An empty source label name is invalid -- TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "host", "-", ""); +-- Both hosts are joined to the same `host` at 0s, leaving two series with the same label set -- +TQL EVAL (0, 15, '5s') label_join(test, "host", "-", "idc"); + -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "new_host", "-", "idc", "host"); @@ -50,8 +56,13 @@ TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "new_idc", "$2", "idc", -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "idc", "$2", "idc", "(.*):(.*)"); +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "idc", "$2", "idc", "(.*):(.*)"); + +-- Both hosts are rewritten to the same `host`, leaving two series with the same label set -- +TQL EVAL (0, 15, '5s') label_replace(test, "host", "x", "host", ".*"); + -- test the empty source label -- --- TODO(dennis): we can't remove the label currently -- -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host2"}, "idc2", "", "", ""); @@ -77,6 +88,10 @@ TQL EVAL(0, 15, '5s') label_replace(test{host="host1"}, "host2", "host2", "insta -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') label_replace(test{host="host1"}, "host2", "", "instance", ""); +-- An empty replacement removes an existing label -- +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') label_replace(test{idc="idc1"}, "idc", "", "instance", ""); + -- Empty regex and different label value -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} * label_replace(vector(1), "host", "host2", "host", ""); diff --git a/tests/cases/standalone/common/promql/native_time_selection.result b/tests/cases/standalone/common/promql/native_time_selection.result index 2c0e20847cb..945189e68f6 100644 --- a/tests/cases/standalone/common/promql/native_time_selection.result +++ b/tests/cases/standalone/common/promql/native_time_selection.result @@ -613,6 +613,15 @@ TQL EVAL (1, 1, '1s', '1s') native_time_sec offset -500ms; | 11.0 | 1970-01-01T00:00:01 | +------+---------------------+ +-- `timestamp()` reports the timestamp of the selected sample (1s), without the offset applied. +TQL EVAL (1, 1, '1s', '1s') timestamp(native_time_sec offset -500ms); + ++---------------------+-------+ +| ts | value | ++---------------------+-------+ +| 1970-01-01T00:00:01 | 1.0 | ++---------------------+-------+ + DROP TABLE native_time_sec; Affected Rows: 0 diff --git a/tests/cases/standalone/common/promql/native_time_selection.sql b/tests/cases/standalone/common/promql/native_time_selection.sql index ca795b0b3ad..204b047ef23 100644 --- a/tests/cases/standalone/common/promql/native_time_selection.sql +++ b/tests/cases/standalone/common/promql/native_time_selection.sql @@ -219,4 +219,6 @@ CREATE TABLE native_time_sec (ts TIMESTAMP(0) TIME INDEX, val DOUBLE); INSERT INTO native_time_sec VALUES (0, 10), (1, 11), (2, 12); TQL EVAL (1, 1, '1s', '1s') native_time_sec offset 500ms; TQL EVAL (1, 1, '1s', '1s') native_time_sec offset -500ms; +-- `timestamp()` reports the timestamp of the selected sample (1s), without the offset applied. +TQL EVAL (1, 1, '1s', '1s') timestamp(native_time_sec offset -500ms); DROP TABLE native_time_sec; diff --git a/tests/cases/standalone/common/promql/timestamp_fn.result b/tests/cases/standalone/common/promql/timestamp_fn.result index 1f5682649a4..fc4e9c46fa6 100644 --- a/tests/cases/standalone/common/promql/timestamp_fn.result +++ b/tests/cases/standalone/common/promql/timestamp_fn.result @@ -113,17 +113,17 @@ tql eval (0, 60, '30s') abs(timestamp(timestamp_test) - avg(timestamp(timestamp_ -- Test Issue 6707 tql eval timestamp(demo_memory_usage_bytes * 1); -+------+--------------------+ -| time | value * Float64(1) | -+------+--------------------+ -+------+--------------------+ ++------+-------+ +| time | value | ++------+-------+ ++------+-------+ tql eval timestamp(-demo_memory_usage_bytes); -+------+-----------+ -| time | (- value) | -+------+-----------+ -+------+-----------+ ++------+-------+ +| time | value | ++------+-------+ ++------+-------+ tql eval (0, 60, '30s') timestamp(timestamp_test) == 60; @@ -133,6 +133,79 @@ tql eval (0, 60, '30s') timestamp(timestamp_test) == 60; | 1970-01-01T00:01:00 | 60.0 | +---------------------+-------+ +-- timestamp() reports the timestamp of the selected sample, not the evaluation time +tql eval (60, 60, '1s') timestamp(timestamp_test offset 30s); + ++---------------------+-------+ +| ts | value | ++---------------------+-------+ +| 1970-01-01T00:01:00 | 1.0 | ++---------------------+-------+ + +tql eval (60, 60, '1s') timestamp((timestamp_test offset 30s)); + ++---------------------+-------+ +| ts | value | ++---------------------+-------+ +| 1970-01-01T00:01:00 | 1.0 | ++---------------------+-------+ + +-- The result of a function is sampled at the evaluation time +tql eval (90, 90, '1s') timestamp(abs(timestamp_test)); + ++---------------------+-------+ +| ts | value | ++---------------------+-------+ +| 1970-01-01T00:01:30 | 90.0 | ++---------------------+-------+ + +-- A row whose fields are all NULL holds no sample and gets no timestamp; a row with any +-- non-NULL field does. +create table timestamp_null (ts timestamp time index, host string primary key, a double, b double); + +Affected Rows: 0 + +insert into timestamp_null values + (0, 'none', NULL, NULL), + (0, 'only_a', 1.0, NULL), + (0, 'only_b', NULL, 2.0); + +Affected Rows: 3 + +-- SQLNESS SORT_RESULT 3 1 +tql eval (60, 60, '1s') timestamp(timestamp_null); + ++---------------------+-------+--------+ +| ts | value | host | ++---------------------+-------+--------+ +| 1970-01-01T00:01:00 | 0.0 | only_a | +| 1970-01-01T00:01:00 | 0.0 | only_b | ++---------------------+-------+--------+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (60, 60, '1s') timestamp(-timestamp_null); + ++---------------------+-------+--------+ +| ts | value | host | ++---------------------+-------+--------+ +| 1970-01-01T00:01:00 | 60.0 | only_a | +| 1970-01-01T00:01:00 | 60.0 | only_b | ++---------------------+-------+--------+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (60, 60, '1s') timestamp(abs(timestamp_null)); + ++---------------------+-------+--------+ +| ts | value | host | ++---------------------+-------+--------+ +| 1970-01-01T00:01:00 | 60.0 | only_a | +| 1970-01-01T00:01:00 | 60.0 | only_b | ++---------------------+-------+--------+ + +drop table timestamp_null; + +Affected Rows: 0 + -- Test timestamp() with multiple metrics create table timestamp_test2 (ts timestamp time index, val double); diff --git a/tests/cases/standalone/common/promql/timestamp_fn.sql b/tests/cases/standalone/common/promql/timestamp_fn.sql index 123bdeb5881..cc307007e4b 100644 --- a/tests/cases/standalone/common/promql/timestamp_fn.sql +++ b/tests/cases/standalone/common/promql/timestamp_fn.sql @@ -43,6 +43,34 @@ tql eval timestamp(-demo_memory_usage_bytes); tql eval (0, 60, '30s') timestamp(timestamp_test) == 60; +-- timestamp() reports the timestamp of the selected sample, not the evaluation time +tql eval (60, 60, '1s') timestamp(timestamp_test offset 30s); + +tql eval (60, 60, '1s') timestamp((timestamp_test offset 30s)); + +-- The result of a function is sampled at the evaluation time +tql eval (90, 90, '1s') timestamp(abs(timestamp_test)); + +-- A row whose fields are all NULL holds no sample and gets no timestamp; a row with any +-- non-NULL field does. +create table timestamp_null (ts timestamp time index, host string primary key, a double, b double); + +insert into timestamp_null values + (0, 'none', NULL, NULL), + (0, 'only_a', 1.0, NULL), + (0, 'only_b', NULL, 2.0); + +-- SQLNESS SORT_RESULT 3 1 +tql eval (60, 60, '1s') timestamp(timestamp_null); + +-- SQLNESS SORT_RESULT 3 1 +tql eval (60, 60, '1s') timestamp(-timestamp_null); + +-- SQLNESS SORT_RESULT 3 1 +tql eval (60, 60, '1s') timestamp(abs(timestamp_null)); + +drop table timestamp_null; + -- Test timestamp() with multiple metrics create table timestamp_test2 (ts timestamp time index, val double);