From b27a97ab2e1ce0e9c85939b6a49dbe710335a77c Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:30:29 +0800 Subject: [PATCH] fix(servers): validate remote write native histograms (#8654) Signed-off-by: shuiyisong --- src/common/query/src/native_histogram.rs | 2 + src/servers/src/prom_remote_write/v2.rs | 606 ++++++++++++++++++++++- tests-integration/tests/http.rs | 7 +- 3 files changed, 587 insertions(+), 28 deletions(-) diff --git a/src/common/query/src/native_histogram.rs b/src/common/query/src/native_histogram.rs index 9d07db19c32..9ffcccb7def 100644 --- a/src/common/query/src/native_histogram.rs +++ b/src/common/query/src/native_histogram.rs @@ -110,6 +110,8 @@ pub fn is_native_histogram_value_type(data_type: &ConcreteDataType) -> bool { data_type == native_histogram_value_type() } +pub const CUSTOM_BUCKETS_SCHEMA: i32 = -53; + // --------------------------------------------------------------------------- // Stable Parquet field ids for native-histogram sub-fields. // diff --git a/src/servers/src/prom_remote_write/v2.rs b/src/servers/src/prom_remote_write/v2.rs index ceb3d67d152..923f860be38 100644 --- a/src/servers/src/prom_remote_write/v2.rs +++ b/src/servers/src/prom_remote_write/v2.rs @@ -16,9 +16,11 @@ use std::collections::hash_map::Entry; use ahash::{HashMap, HashMapExt, HashSet, HashSetExt}; use api::greptime_proto::io::prometheus::write::v2::histogram::{Count, ZeroCount}; +use api::greptime_proto::io::prometheus::write::v2::{ + BucketSpan, Histogram, Request, Sample, TimeSeries, +}; #[cfg(test)] use api::greptime_proto::io::prometheus::write::v2::{Exemplar, Metadata, metadata}; -use api::greptime_proto::io::prometheus::write::v2::{Histogram, Request, Sample, TimeSeries}; use api::helper::ColumnDataTypeWrapper; use api::v1::value::ValueData; use api::v1::{ColumnSchema, ListValue, RowInsertRequest, Rows, SemanticType, Value}; @@ -43,6 +45,9 @@ use crate::row_writer::{self, TableData}; type PromTags = Vec<(String, String)>; type ResolvedSeriesLabels = (PromCtx, String, PromTags); +const MIN_REMOTE_WRITE_V2_SCHEMA: i32 = -4; +const MAX_REMOTE_WRITE_V2_SCHEMA: i32 = 8; +const MAX_REDUCIBLE_REMOTE_WRITE_V2_SCHEMA: i32 = 52; pub(crate) fn decode_remote_write_v2_request(is_zstd: bool, body: Bytes) -> Result { let _timer = crate::metrics::METRIC_HTTP_PROM_STORE_DECODE_ELAPSED.start_timer(); @@ -279,6 +284,9 @@ fn native_histogram_column_schema() -> ColumnSchema { } fn native_histogram_struct_value(histogram: &Histogram) -> Result { + let uses_float_counts = native_histogram_uses_float_counts(histogram)?; + validate_native_histogram(histogram, uses_float_counts)?; + let mut items = Vec::with_capacity(NATIVE_HISTOGRAM_FIELD_NAMES.len()); items.extend([ pb_value(ValueData::I32Value(histogram.schema)), @@ -295,39 +303,41 @@ fn native_histogram_struct_value(histogram: &Histogram) -> Result { u32_list_value(histogram.negative_spans.iter().map(|span| span.length)), ]); - let int_counts = match (&histogram.count, &histogram.zero_count) { - (Some(Count::CountInt(count)), Some(ZeroCount::ZeroCountInt(zero_count))) => { - (*count, *zero_count) - } - (Some(Count::CountInt(count)), _) => (*count, 0), - (_, Some(ZeroCount::ZeroCountInt(zero_count))) => (0, *zero_count), - _ => (0, 0), - }; - let float_counts = match (&histogram.count, &histogram.zero_count) { - (Some(Count::CountFloat(count)), Some(ZeroCount::ZeroCountFloat(zero_count))) => { - Some((*count, *zero_count)) - } - (Some(Count::CountFloat(count)), _) => Some((*count, 0.0)), - _ => None, - }; - - if let Some(counts) = float_counts { + if uses_float_counts { + validate_float_native_histogram_counts(histogram)?; + let count = match histogram.count.as_ref() { + Some(Count::CountFloat(count)) => *count, + _ => 0.0, + }; + let zero_count = match histogram.zero_count.as_ref() { + Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count, + _ => 0.0, + }; items.extend([ null_pb_value(), null_pb_value(), i64_list_value(std::iter::empty()), i64_list_value(std::iter::empty()), - pb_value(ValueData::F64Value(counts.0)), - pb_value(ValueData::F64Value(counts.1)), + pb_value(ValueData::F64Value(count)), + pb_value(ValueData::F64Value(zero_count)), f64_list_value(histogram.positive_counts.iter().copied()), f64_list_value(histogram.negative_counts.iter().copied()), ]); } else { + let count = match histogram.count.as_ref() { + Some(Count::CountInt(count)) => *count, + _ => 0, + }; + let zero_count = match histogram.zero_count.as_ref() { + Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count, + _ => 0, + }; let positive_buckets = bucket_counts_from_deltas(&histogram.positive_deltas)?; let negative_buckets = bucket_counts_from_deltas(&histogram.negative_deltas)?; + validate_integer_native_histogram_counts(histogram, &positive_buckets, &negative_buckets)?; items.extend([ - pb_value(ValueData::U64Value(int_counts.0)), - pb_value(ValueData::U64Value(int_counts.1)), + pb_value(ValueData::U64Value(count)), + pb_value(ValueData::U64Value(zero_count)), i64_list_value(positive_buckets.iter().copied()), i64_list_value(negative_buckets.iter().copied()), null_pb_value(), @@ -340,6 +350,329 @@ fn native_histogram_struct_value(histogram: &Histogram) -> Result { Ok(ValueData::StructValue(api::v1::StructValue { items })) } +fn validate_native_histogram(histogram: &Histogram, uses_float_counts: bool) -> Result<()> { + validate_native_histogram_schema(histogram.schema)?; + validate_native_histogram_custom_values(histogram)?; + + if histogram.schema == CUSTOM_BUCKETS_SCHEMA { + ensure!( + histogram.zero_threshold == 0.0 && native_histogram_zero_count_is_zero(histogram), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 custom native histogram must not use a zero bucket" + .to_string(), + } + ); + ensure!( + histogram.negative_spans.is_empty() + && histogram.negative_deltas.is_empty() + && histogram.negative_counts.is_empty(), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 custom native histogram must not use negative buckets" + .to_string(), + } + ); + } + + let (positive_buckets, negative_buckets) = if uses_float_counts { + ( + histogram.positive_counts.len(), + histogram.negative_counts.len(), + ) + } else { + ( + histogram.positive_deltas.len(), + histogram.negative_deltas.len(), + ) + }; + let custom_max_index = if histogram.schema == CUSTOM_BUCKETS_SCHEMA { + Some( + i32::try_from(histogram.custom_values.len()).ok().context( + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 custom native histogram has too many custom_values" + .to_string(), + }, + )?, + ) + } else { + None + }; + validate_native_histogram_spans( + "positive", + &histogram.positive_spans, + positive_buckets, + custom_max_index, + )?; + validate_native_histogram_spans( + "negative", + &histogram.negative_spans, + negative_buckets, + None, + )?; + + Ok(()) +} + +fn validate_native_histogram_schema(schema: i32) -> Result<()> { + if schema == CUSTOM_BUCKETS_SCHEMA + || (MIN_REMOTE_WRITE_V2_SCHEMA..=MAX_REMOTE_WRITE_V2_SCHEMA).contains(&schema) + { + return Ok(()); + } + + if (MAX_REMOTE_WRITE_V2_SCHEMA + 1..=MAX_REDUCIBLE_REMOTE_WRITE_V2_SCHEMA).contains(&schema) { + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram schema {schema} must be reduced before ingestion" + ), + } + .fail() + } else { + error::InvalidPromRemoteRequestSnafu { + msg: format!("remote write v2 native histogram schema {schema} is unsupported"), + } + .fail() + } +} + +fn validate_native_histogram_custom_values(histogram: &Histogram) -> Result<()> { + if histogram.schema != CUSTOM_BUCKETS_SCHEMA { + ensure!( + histogram.custom_values.is_empty(), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 standard native histogram must not use custom_values" + .to_string(), + } + ); + return Ok(()); + } + + for value in &histogram.custom_values { + ensure!( + !value.is_nan() && *value != f64::INFINITY, + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 custom native histogram custom_values must not contain +Inf or NaN" + .to_string(), + } + ); + } + for values in histogram.custom_values.windows(2) { + ensure!( + values[0] < values[1], + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 custom native histogram custom_values must be sorted" + .to_string(), + } + ); + } + + Ok(()) +} + +fn validate_native_histogram_spans( + name: &str, + spans: &[BucketSpan], + bucket_count: usize, + custom_max_index: Option, +) -> Result<()> { + let span_len = spans + .iter() + .try_fold(0usize, |sum, span| sum.checked_add(span.length as usize)) + .context(error::InvalidPromRemoteRequestSnafu { + msg: format!("remote write v2 native histogram {name} spans overflow"), + })?; + ensure!( + span_len == bucket_count, + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram {name} spans describe {span_len} buckets, found {bucket_count}" + ), + } + ); + + let mut current_index = 0i32; + for (span_index, span) in spans.iter().enumerate() { + ensure!( + span.offset >= 0 || (span_index == 0 && custom_max_index.is_none()), + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram {name} span {} has negative offset {}", + span_index + 1, + span.offset + ), + } + ); + current_index = if span_index == 0 { + span.offset + } else { + current_index.checked_add(span.offset).context( + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram {name} span index overflows i32" + ), + }, + )? + }; + + for _ in 0..span.length { + if let Some(max_index) = custom_max_index { + ensure!( + (0..=max_index).contains(¤t_index), + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 custom native histogram {name} bucket index {current_index} is out of range" + ), + } + ); + } + current_index = + current_index + .checked_add(1) + .context(error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram {name} span index overflows i32" + ), + })?; + } + } + + Ok(()) +} + +fn validate_float_native_histogram_counts(histogram: &Histogram) -> Result<()> { + let count = match histogram.count.as_ref() { + Some(Count::CountFloat(count)) => *count, + _ => 0.0, + }; + ensure!( + count >= 0.0 || count.is_nan(), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 native histogram float count must not be negative".to_string(), + } + ); + + let zero_count = match histogram.zero_count.as_ref() { + Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count, + _ => 0.0, + }; + ensure!( + zero_count >= 0.0 || zero_count.is_nan(), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 native histogram float zero_count must not be negative" + .to_string(), + } + ); + + for (name, counts) in [ + ("positive", &histogram.positive_counts), + ("negative", &histogram.negative_counts), + ] { + for (index, count) in counts.iter().enumerate() { + ensure!( + *count >= 0.0 || count.is_nan(), + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram {name} bucket {} count must not be negative", + index + 1 + ), + } + ); + } + } + + Ok(()) +} + +fn validate_integer_native_histogram_counts( + histogram: &Histogram, + positive_buckets: &[i64], + negative_buckets: &[i64], +) -> Result<()> { + let count = match histogram.count.as_ref() { + Some(Count::CountInt(count)) => *count, + _ => 0, + }; + let zero_count = match histogram.zero_count.as_ref() { + Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count, + _ => 0, + }; + let bucket_count = positive_buckets + .iter() + .chain(negative_buckets) + .try_fold(zero_count, |total, bucket| { + total.checked_add(*bucket as u64) + }) + .context(error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 native histogram bucket total overflows u64".to_string(), + })?; + ensure!( + if histogram.sum.is_nan() { + bucket_count <= count + } else { + bucket_count == count + }, + error::InvalidPromRemoteRequestSnafu { + msg: format!( + "remote write v2 native histogram has {bucket_count} observations in buckets, count is {count}" + ), + } + ); + + Ok(()) +} + +fn native_histogram_zero_count_is_zero(histogram: &Histogram) -> bool { + match histogram.zero_count.as_ref() { + Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count == 0, + Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count == 0.0, + None => true, + } +} + +fn native_histogram_uses_float_counts(histogram: &Histogram) -> Result { + let uses_float_count = matches!(histogram.count, Some(Count::CountFloat(_))) + || matches!(histogram.zero_count, Some(ZeroCount::ZeroCountFloat(_))); + let uses_int_count = matches!(histogram.count, Some(Count::CountInt(_))) + || matches!(histogram.zero_count, Some(ZeroCount::ZeroCountInt(_))); + let uses_float_buckets = + !histogram.positive_counts.is_empty() || !histogram.negative_counts.is_empty(); + let uses_int_buckets = + !histogram.positive_deltas.is_empty() || !histogram.negative_deltas.is_empty(); + + if matches!( + (&histogram.count, &histogram.zero_count), + (Some(Count::CountInt(_)), Some(ZeroCount::ZeroCountFloat(_))) + | (Some(Count::CountFloat(_)), Some(ZeroCount::ZeroCountInt(_))) + ) { + return error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 native histogram count and zero_count must use the same integer or float family".to_string(), + } + .fail(); + } + + ensure!( + !(uses_float_buckets && uses_int_buckets), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 native histogram bucket counts must use either integer deltas or float counts".to_string(), + } + ); + ensure!( + !(uses_float_count && uses_int_buckets), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 float native histogram must not use integer bucket deltas" + .to_string(), + } + ); + ensure!( + !(uses_int_count && uses_float_buckets), + error::InvalidPromRemoteRequestSnafu { + msg: "remote write v2 integer native histogram must not use float bucket counts" + .to_string(), + } + ); + + Ok(uses_float_count || uses_float_buckets) +} + fn pb_value(value_data: ValueData) -> Value { optional_pb_value(Some(value_data)) } @@ -859,11 +1192,229 @@ mod tests { "conflicts with an internal native histogram label", )); + cases.push(( + "int count with float zero count", + request_with_histogram(Histogram { + count: Some(Count::CountInt(1)), + zero_count: Some(ZeroCount::ZeroCountFloat(0.5)), + ..Default::default() + }), + "count and zero_count must use the same integer or float family", + )); + + cases.push(( + "float count with int zero count", + request_with_histogram(Histogram { + count: Some(Count::CountFloat(1.0)), + zero_count: Some(ZeroCount::ZeroCountInt(1)), + ..Default::default() + }), + "count and zero_count must use the same integer or float family", + )); + + cases.push(( + "reducible schema", + request_with_histogram(Histogram { + schema: 9, + ..Default::default() + }), + "schema 9 must be reduced before ingestion", + )); + + cases.push(( + "unsupported schema", + request_with_histogram(Histogram { + schema: 53, + ..Default::default() + }), + "schema 53 is unsupported", + )); + + cases.push(( + "standard schema with custom values", + request_with_histogram(Histogram { + schema: 1, + custom_values: vec![1.0], + ..Default::default() + }), + "standard native histogram must not use custom_values", + )); + + cases.push(( + "custom values with inf", + request_with_histogram(Histogram { + schema: CUSTOM_BUCKETS_SCHEMA, + custom_values: vec![f64::INFINITY], + ..Default::default() + }), + "custom_values must not contain +Inf or NaN", + )); + + cases.push(( + "custom values not sorted", + request_with_histogram(Histogram { + schema: CUSTOM_BUCKETS_SCHEMA, + custom_values: vec![2.0, 1.0], + ..Default::default() + }), + "custom_values must be sorted", + )); + + cases.push(( + "custom schema with zero bucket", + request_with_histogram(Histogram { + schema: CUSTOM_BUCKETS_SCHEMA, + zero_count: Some(ZeroCount::ZeroCountInt(1)), + ..Default::default() + }), + "custom native histogram must not use a zero bucket", + )); + + cases.push(( + "custom schema with negative buckets", + request_with_histogram(Histogram { + schema: CUSTOM_BUCKETS_SCHEMA, + negative_spans: vec![BucketSpan { + offset: -1, + length: 1, + }], + negative_deltas: vec![1], + ..Default::default() + }), + "custom native histogram must not use negative buckets", + )); + + cases.push(( + "span count mismatch", + request_with_histogram(Histogram { + positive_spans: vec![BucketSpan { + offset: 0, + length: 2, + }], + positive_deltas: vec![1], + ..Default::default() + }), + "positive spans describe 2 buckets, found 1", + )); + + cases.push(( + "negative offset after first span", + request_with_histogram(Histogram { + count: Some(Count::CountInt(2)), + positive_spans: vec![ + BucketSpan { + offset: 0, + length: 1, + }, + BucketSpan { + offset: -1, + length: 1, + }, + ], + positive_deltas: vec![1, 0], + ..Default::default() + }), + "positive span 2 has negative offset -1", + )); + + cases.push(( + "negative custom span offset", + request_with_histogram(Histogram { + count: Some(Count::CountInt(1)), + schema: CUSTOM_BUCKETS_SCHEMA, + custom_values: vec![1.0], + positive_spans: vec![BucketSpan { + offset: -1, + length: 1, + }], + positive_deltas: vec![1], + ..Default::default() + }), + "positive span 1 has negative offset -1", + )); + + cases.push(( + "integer bucket total mismatch", + request_with_histogram(Histogram { + count: Some(Count::CountInt(0)), + positive_spans: vec![BucketSpan { + offset: 0, + length: 1, + }], + positive_deltas: vec![1], + ..Default::default() + }), + "has 1 observations in buckets, count is 0", + )); + + cases.push(( + "negative float count", + request_with_histogram(Histogram { + count: Some(Count::CountFloat(-1.0)), + ..Default::default() + }), + "float count must not be negative", + )); + + cases.push(( + "negative float zero count", + request_with_histogram(Histogram { + count: Some(Count::CountFloat(0.0)), + zero_count: Some(ZeroCount::ZeroCountFloat(-1.0)), + ..Default::default() + }), + "float zero_count must not be negative", + )); + + cases.push(( + "negative float bucket count", + request_with_histogram(Histogram { + count: Some(Count::CountFloat(0.0)), + positive_spans: vec![BucketSpan { + offset: 0, + length: 1, + }], + positive_counts: vec![-1.0], + ..Default::default() + }), + "positive bucket 1 count must not be negative", + )); + + cases.push(( + "custom span index out of range", + request_with_histogram(Histogram { + schema: CUSTOM_BUCKETS_SCHEMA, + custom_values: vec![1.0], + positive_spans: vec![BucketSpan { + offset: 2, + length: 1, + }], + positive_deltas: vec![1], + ..Default::default() + }), + "positive bucket index 2 is out of range", + )); + for (name, request, expected) in cases { assert_invalid(name, request, expected); } } + #[test] + fn test_into_context_req_allows_nan_observations_outside_buckets() { + into_write_requests(request_with_histogram(Histogram { + count: Some(Count::CountInt(2)), + sum: f64::NAN, + positive_spans: vec![BucketSpan { + offset: 0, + length: 1, + }], + positive_deltas: vec![1], + ..Default::default() + })) + .unwrap(); + } + #[test] fn test_into_context_req_allows_empty_label_values() { let ctx_req = into_write_requests(test_util::request_with_labels_and_samples( @@ -1040,7 +1591,7 @@ mod tests { #[test] fn test_into_context_req_converts_int_and_float_histograms_to_one_schema() { let float_histogram = Histogram { - count: Some(api::greptime_proto::io::prometheus::write::v2::histogram::Count::CountFloat(3.5)), + count: Some(api::greptime_proto::io::prometheus::write::v2::histogram::Count::CountFloat(6.0)), zero_count: Some( api::greptime_proto::io::prometheus::write::v2::histogram::ZeroCount::ZeroCountFloat( 0.5, @@ -1107,7 +1658,7 @@ mod tests { assert_eq!(histogram_field_value(&rows, 1, COUNT_U64_FIELD), None); assert_eq!( histogram_field_value(&rows, 1, COUNT_F64_FIELD), - Some(ValueData::F64Value(3.5)) + Some(ValueData::F64Value(6.0)) ); assert!(is_empty_list(histogram_field_value( &rows, @@ -1131,6 +1682,13 @@ mod tests { ) } + fn request_with_histogram(histogram: Histogram) -> Request { + test_util::request_with_labels_and_histograms( + vec![(METRIC_NAME_LABEL, "metric")], + vec![histogram], + ) + } + fn assert_invalid(name: &str, request: Request, expected: &str) { let err = match into_write_requests(request) { Ok(_) => panic!("{name}: expected invalid request error"), diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 06a37b46440..99ef6bc59e7 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -2702,7 +2702,7 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag ], histograms: vec![ Histogram { - count: Some(Count::CountInt(4)), + count: Some(Count::CountInt(8)), sum: 10.0, schema: 1, zero_threshold: 0.001, @@ -2720,11 +2720,10 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag reset_hint: 2, timestamp: 3000, start_timestamp: 1500, - custom_values: vec![0.5, 1.5], ..Default::default() }, Histogram { - count: Some(Count::CountFloat(3.5)), + count: Some(Count::CountFloat(6.0)), sum: 20.0, schema: 2, zero_threshold: 0.002, @@ -2773,7 +2772,7 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag "prometheus_remote_write_v2_native_histogram_rows", &client, "select greptime_timestamp, greptime_native_histogram, job, instance from remote_write_v2_latency_seconds order by greptime_timestamp;", - "[[3000,{\"count_f64\":null,\"count_u64\":4,\"custom_values\":[0.5,1.5],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_u64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":3.5,\"count_u64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_u64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]", + "[[3000,{\"count_f64\":null,\"count_u64\":8,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_u64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":6.0,\"count_u64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_u64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]", ) .await;