rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-08 21:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(servers): validate remote write native histograms (#8654)

Browse Source 
Signed-off-by: shuiyisong <xixing.sys@gmail.com>
This commit is contained in:
shuiyisong
2026-07-27 10:30:29 +00:00
committed by GitHub
parent b462d5d19e
commit b27a97ab2e
3 changed files with 587 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/query/src/native_histogram.rs
+2
View File
@@ -110,6 +110,8 @@ pub fn is_native_histogram_value_type(data_type: &ConcreteDataType) -> bool {
data_type == native_histogram_value_type()
}
pub const CUSTOM_BUCKETS_SCHEMA: i32 = -53;
// ---------------------------------------------------------------------------
// Stable Parquet field ids for native-histogram sub-fields.
//
src/servers/src/prom_remote_write/v2.rs
+582 -24
View File
@@ -16,9 +16,11 @@ use std::collections::hash_map::Entry;
use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
use api::greptime_proto::io::prometheus::write::v2::histogram::{Count, ZeroCount};
use api::greptime_proto::io::prometheus::write::v2::{
BucketSpan, Histogram, Request, Sample, TimeSeries,
};
#[cfg(test)]
use api::greptime_proto::io::prometheus::write::v2::{Exemplar, Metadata, metadata};
use api::greptime_proto::io::prometheus::write::v2::{Histogram, Request, Sample, TimeSeries};
use api::helper::ColumnDataTypeWrapper;
use api::v1::value::ValueData;
use api::v1::{ColumnSchema, ListValue, RowInsertRequest, Rows, SemanticType, Value};
@@ -43,6 +45,9 @@ use crate::row_writer::{self, TableData};
type PromTags = Vec<(String, String)>;
type ResolvedSeriesLabels = (PromCtx, String, PromTags);
const MIN_REMOTE_WRITE_V2_SCHEMA: i32 = -4;
const MAX_REMOTE_WRITE_V2_SCHEMA: i32 = 8;
const MAX_REDUCIBLE_REMOTE_WRITE_V2_SCHEMA: i32 = 52;
pub(crate) fn decode_remote_write_v2_request(is_zstd: bool, body: Bytes) -> Result<Request> {
let _timer = crate::metrics::METRIC_HTTP_PROM_STORE_DECODE_ELAPSED.start_timer();
@@ -279,6 +284,9 @@ fn native_histogram_column_schema() -> ColumnSchema {
}
fn native_histogram_struct_value(histogram: &Histogram) -> Result<ValueData> {
let uses_float_counts = native_histogram_uses_float_counts(histogram)?;
validate_native_histogram(histogram, uses_float_counts)?;
let mut items = Vec::with_capacity(NATIVE_HISTOGRAM_FIELD_NAMES.len());
items.extend([
pb_value(ValueData::I32Value(histogram.schema)),
@@ -295,39 +303,41 @@ fn native_histogram_struct_value(histogram: &Histogram) -> Result<ValueData> {
u32_list_value(histogram.negative_spans.iter().map(|span| span.length)),
]);
let int_counts = match (&histogram.count, &histogram.zero_count) {
(Some(Count::CountInt(count)), Some(ZeroCount::ZeroCountInt(zero_count))) => {
(*count, *zero_count)
}
(Some(Count::CountInt(count)), _) => (*count, 0),
(_, Some(ZeroCount::ZeroCountInt(zero_count))) => (0, *zero_count),
_ => (0, 0),
};
let float_counts = match (&histogram.count, &histogram.zero_count) {
(Some(Count::CountFloat(count)), Some(ZeroCount::ZeroCountFloat(zero_count))) => {
Some((*count, *zero_count))
}
(Some(Count::CountFloat(count)), _) => Some((*count, 0.0)),
_ => None,
};
if let Some(counts) = float_counts {
if uses_float_counts {
validate_float_native_histogram_counts(histogram)?;
let count = match histogram.count.as_ref() {
Some(Count::CountFloat(count)) => *count,
_ => 0.0,
};
let zero_count = match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count,
_ => 0.0,
};
items.extend([
null_pb_value(),
null_pb_value(),
i64_list_value(std::iter::empty()),
i64_list_value(std::iter::empty()),
pb_value(ValueData::F64Value(counts.0)),
pb_value(ValueData::F64Value(counts.1)),
pb_value(ValueData::F64Value(count)),
pb_value(ValueData::F64Value(zero_count)),
f64_list_value(histogram.positive_counts.iter().copied()),
f64_list_value(histogram.negative_counts.iter().copied()),
]);
} else {
let count = match histogram.count.as_ref() {
Some(Count::CountInt(count)) => *count,
_ => 0,
};
let zero_count = match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count,
_ => 0,
};
let positive_buckets = bucket_counts_from_deltas(&histogram.positive_deltas)?;
let negative_buckets = bucket_counts_from_deltas(&histogram.negative_deltas)?;
validate_integer_native_histogram_counts(histogram, &positive_buckets, &negative_buckets)?;
items.extend([
pb_value(ValueData::U64Value(int_counts.0)),
pb_value(ValueData::U64Value(int_counts.1)),
pb_value(ValueData::U64Value(count)),
pb_value(ValueData::U64Value(zero_count)),
i64_list_value(positive_buckets.iter().copied()),
i64_list_value(negative_buckets.iter().copied()),
null_pb_value(),
@@ -340,6 +350,329 @@ fn native_histogram_struct_value(histogram: &Histogram) -> Result<ValueData> {
Ok(ValueData::StructValue(api::v1::StructValue { items }))
}
fn validate_native_histogram(histogram: &Histogram, uses_float_counts: bool) -> Result<()> {
validate_native_histogram_schema(histogram.schema)?;
validate_native_histogram_custom_values(histogram)?;
if histogram.schema == CUSTOM_BUCKETS_SCHEMA {
ensure!(
histogram.zero_threshold == 0.0 && native_histogram_zero_count_is_zero(histogram),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram must not use a zero bucket"
.to_string(),
}
);
ensure!(
histogram.negative_spans.is_empty()
&& histogram.negative_deltas.is_empty()
&& histogram.negative_counts.is_empty(),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram must not use negative buckets"
.to_string(),
}
);
}
let (positive_buckets, negative_buckets) = if uses_float_counts {
(
histogram.positive_counts.len(),
histogram.negative_counts.len(),
)
} else {
(
histogram.positive_deltas.len(),
histogram.negative_deltas.len(),
)
};
let custom_max_index = if histogram.schema == CUSTOM_BUCKETS_SCHEMA {
Some(
i32::try_from(histogram.custom_values.len()).ok().context(
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram has too many custom_values"
.to_string(),
},
)?,
)
} else {
None
};
validate_native_histogram_spans(
"positive",
&histogram.positive_spans,
positive_buckets,
custom_max_index,
)?;
validate_native_histogram_spans(
"negative",
&histogram.negative_spans,
negative_buckets,
None,
)?;
Ok(())
}
fn validate_native_histogram_schema(schema: i32) -> Result<()> {
if schema == CUSTOM_BUCKETS_SCHEMA
|| (MIN_REMOTE_WRITE_V2_SCHEMA..=MAX_REMOTE_WRITE_V2_SCHEMA).contains(&schema)
{
return Ok(());
}
if (MAX_REMOTE_WRITE_V2_SCHEMA + 1..=MAX_REDUCIBLE_REMOTE_WRITE_V2_SCHEMA).contains(&schema) {
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram schema {schema} must be reduced before ingestion"
),
}
.fail()
} else {
error::InvalidPromRemoteRequestSnafu {
msg: format!("remote write v2 native histogram schema {schema} is unsupported"),
}
.fail()
}
}
fn validate_native_histogram_custom_values(histogram: &Histogram) -> Result<()> {
if histogram.schema != CUSTOM_BUCKETS_SCHEMA {
ensure!(
histogram.custom_values.is_empty(),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 standard native histogram must not use custom_values"
.to_string(),
}
);
return Ok(());
}
for value in &histogram.custom_values {
ensure!(
!value.is_nan() && *value != f64::INFINITY,
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram custom_values must not contain +Inf or NaN"
.to_string(),
}
);
}
for values in histogram.custom_values.windows(2) {
ensure!(
values[0] < values[1],
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 custom native histogram custom_values must be sorted"
.to_string(),
}
);
}
Ok(())
}
fn validate_native_histogram_spans(
name: &str,
spans: &[BucketSpan],
bucket_count: usize,
custom_max_index: Option<i32>,
) -> Result<()> {
let span_len = spans
.iter()
.try_fold(0usize, |sum, span| sum.checked_add(span.length as usize))
.context(error::InvalidPromRemoteRequestSnafu {
msg: format!("remote write v2 native histogram {name} spans overflow"),
})?;
ensure!(
span_len == bucket_count,
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} spans describe {span_len} buckets, found {bucket_count}"
),
}
);
let mut current_index = 0i32;
for (span_index, span) in spans.iter().enumerate() {
ensure!(
span.offset >= 0 || (span_index == 0 && custom_max_index.is_none()),
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} span {} has negative offset {}",
span_index + 1,
span.offset
),
}
);
current_index = if span_index == 0 {
span.offset
} else {
current_index.checked_add(span.offset).context(
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} span index overflows i32"
),
},
)?
};
for _ in 0..span.length {
if let Some(max_index) = custom_max_index {
ensure!(
(0..=max_index).contains(&current_index),
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 custom native histogram {name} bucket index {current_index} is out of range"
),
}
);
}
current_index =
current_index
.checked_add(1)
.context(error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} span index overflows i32"
),
})?;
}
}
Ok(())
}
fn validate_float_native_histogram_counts(histogram: &Histogram) -> Result<()> {
let count = match histogram.count.as_ref() {
Some(Count::CountFloat(count)) => *count,
_ => 0.0,
};
ensure!(
count >= 0.0 || count.is_nan(),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram float count must not be negative".to_string(),
}
);
let zero_count = match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count,
_ => 0.0,
};
ensure!(
zero_count >= 0.0 || zero_count.is_nan(),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram float zero_count must not be negative"
.to_string(),
}
);
for (name, counts) in [
("positive", &histogram.positive_counts),
("negative", &histogram.negative_counts),
] {
for (index, count) in counts.iter().enumerate() {
ensure!(
*count >= 0.0 || count.is_nan(),
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram {name} bucket {} count must not be negative",
index + 1
),
}
);
}
}
Ok(())
}
fn validate_integer_native_histogram_counts(
histogram: &Histogram,
positive_buckets: &[i64],
negative_buckets: &[i64],
) -> Result<()> {
let count = match histogram.count.as_ref() {
Some(Count::CountInt(count)) => *count,
_ => 0,
};
let zero_count = match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count,
_ => 0,
};
let bucket_count = positive_buckets
.iter()
.chain(negative_buckets)
.try_fold(zero_count, |total, bucket| {
total.checked_add(*bucket as u64)
})
.context(error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram bucket total overflows u64".to_string(),
})?;
ensure!(
if histogram.sum.is_nan() {
bucket_count <= count
} else {
bucket_count == count
},
error::InvalidPromRemoteRequestSnafu {
msg: format!(
"remote write v2 native histogram has {bucket_count} observations in buckets, count is {count}"
),
}
);
Ok(())
}
fn native_histogram_zero_count_is_zero(histogram: &Histogram) -> bool {
match histogram.zero_count.as_ref() {
Some(ZeroCount::ZeroCountInt(zero_count)) => *zero_count == 0,
Some(ZeroCount::ZeroCountFloat(zero_count)) => *zero_count == 0.0,
None => true,
}
}
fn native_histogram_uses_float_counts(histogram: &Histogram) -> Result<bool> {
let uses_float_count = matches!(histogram.count, Some(Count::CountFloat(_)))
|| matches!(histogram.zero_count, Some(ZeroCount::ZeroCountFloat(_)));
let uses_int_count = matches!(histogram.count, Some(Count::CountInt(_)))
|| matches!(histogram.zero_count, Some(ZeroCount::ZeroCountInt(_)));
let uses_float_buckets =
!histogram.positive_counts.is_empty() || !histogram.negative_counts.is_empty();
let uses_int_buckets =
!histogram.positive_deltas.is_empty() || !histogram.negative_deltas.is_empty();
if matches!(
(&histogram.count, &histogram.zero_count),
(Some(Count::CountInt(_)), Some(ZeroCount::ZeroCountFloat(_)))
| (Some(Count::CountFloat(_)), Some(ZeroCount::ZeroCountInt(_)))
) {
return error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram count and zero_count must use the same integer or float family".to_string(),
}
.fail();
}
ensure!(
!(uses_float_buckets && uses_int_buckets),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 native histogram bucket counts must use either integer deltas or float counts".to_string(),
}
);
ensure!(
!(uses_float_count && uses_int_buckets),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 float native histogram must not use integer bucket deltas"
.to_string(),
}
);
ensure!(
!(uses_int_count && uses_float_buckets),
error::InvalidPromRemoteRequestSnafu {
msg: "remote write v2 integer native histogram must not use float bucket counts"
.to_string(),
}
);
Ok(uses_float_count || uses_float_buckets)
}
fn pb_value(value_data: ValueData) -> Value {
optional_pb_value(Some(value_data))
}
@@ -859,11 +1192,229 @@ mod tests {
"conflicts with an internal native histogram label",
));
cases.push((
"int count with float zero count",
request_with_histogram(Histogram {
count: Some(Count::CountInt(1)),
zero_count: Some(ZeroCount::ZeroCountFloat(0.5)),
..Default::default()
}),
"count and zero_count must use the same integer or float family",
));
cases.push((
"float count with int zero count",
request_with_histogram(Histogram {
count: Some(Count::CountFloat(1.0)),
zero_count: Some(ZeroCount::ZeroCountInt(1)),
..Default::default()
}),
"count and zero_count must use the same integer or float family",
));
cases.push((
"reducible schema",
request_with_histogram(Histogram {
schema: 9,
..Default::default()
}),
"schema 9 must be reduced before ingestion",
));
cases.push((
"unsupported schema",
request_with_histogram(Histogram {
schema: 53,
..Default::default()
}),
"schema 53 is unsupported",
));
cases.push((
"standard schema with custom values",
request_with_histogram(Histogram {
schema: 1,
custom_values: vec![1.0],
..Default::default()
}),
"standard native histogram must not use custom_values",
));
cases.push((
"custom values with inf",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![f64::INFINITY],
..Default::default()
}),
"custom_values must not contain +Inf or NaN",
));
cases.push((
"custom values not sorted",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![2.0, 1.0],
..Default::default()
}),
"custom_values must be sorted",
));
cases.push((
"custom schema with zero bucket",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
zero_count: Some(ZeroCount::ZeroCountInt(1)),
..Default::default()
}),
"custom native histogram must not use a zero bucket",
));
cases.push((
"custom schema with negative buckets",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
negative_spans: vec![BucketSpan {
offset: -1,
length: 1,
}],
negative_deltas: vec![1],
..Default::default()
}),
"custom native histogram must not use negative buckets",
));
cases.push((
"span count mismatch",
request_with_histogram(Histogram {
positive_spans: vec![BucketSpan {
offset: 0,
length: 2,
}],
positive_deltas: vec![1],
..Default::default()
}),
"positive spans describe 2 buckets, found 1",
));
cases.push((
"negative offset after first span",
request_with_histogram(Histogram {
count: Some(Count::CountInt(2)),
positive_spans: vec![
BucketSpan {
offset: 0,
length: 1,
},
BucketSpan {
offset: -1,
length: 1,
},
],
positive_deltas: vec![1, 0],
..Default::default()
}),
"positive span 2 has negative offset -1",
));
cases.push((
"negative custom span offset",
request_with_histogram(Histogram {
count: Some(Count::CountInt(1)),
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![1.0],
positive_spans: vec![BucketSpan {
offset: -1,
length: 1,
}],
positive_deltas: vec![1],
..Default::default()
}),
"positive span 1 has negative offset -1",
));
cases.push((
"integer bucket total mismatch",
request_with_histogram(Histogram {
count: Some(Count::CountInt(0)),
positive_spans: vec![BucketSpan {
offset: 0,
length: 1,
}],
positive_deltas: vec![1],
..Default::default()
}),
"has 1 observations in buckets, count is 0",
));
cases.push((
"negative float count",
request_with_histogram(Histogram {
count: Some(Count::CountFloat(-1.0)),
..Default::default()
}),
"float count must not be negative",
));
cases.push((
"negative float zero count",
request_with_histogram(Histogram {
count: Some(Count::CountFloat(0.0)),
zero_count: Some(ZeroCount::ZeroCountFloat(-1.0)),
..Default::default()
}),
"float zero_count must not be negative",
));
cases.push((
"negative float bucket count",
request_with_histogram(Histogram {
count: Some(Count::CountFloat(0.0)),
positive_spans: vec![BucketSpan {
offset: 0,
length: 1,
}],
positive_counts: vec![-1.0],
..Default::default()
}),
"positive bucket 1 count must not be negative",
));
cases.push((
"custom span index out of range",
request_with_histogram(Histogram {
schema: CUSTOM_BUCKETS_SCHEMA,
custom_values: vec![1.0],
positive_spans: vec![BucketSpan {
offset: 2,
length: 1,
}],
positive_deltas: vec![1],
..Default::default()
}),
"positive bucket index 2 is out of range",
));
for (name, request, expected) in cases {
assert_invalid(name, request, expected);
}
}
#[test]
fn test_into_context_req_allows_nan_observations_outside_buckets() {
into_write_requests(request_with_histogram(Histogram {
count: Some(Count::CountInt(2)),
sum: f64::NAN,
positive_spans: vec![BucketSpan {
offset: 0,
length: 1,
}],
positive_deltas: vec![1],
..Default::default()
}))
.unwrap();
}
#[test]
fn test_into_context_req_allows_empty_label_values() {
let ctx_req = into_write_requests(test_util::request_with_labels_and_samples(
@@ -1040,7 +1591,7 @@ mod tests {
#[test]
fn test_into_context_req_converts_int_and_float_histograms_to_one_schema() {
let float_histogram = Histogram {
count: Some(api::greptime_proto::io::prometheus::write::v2::histogram::Count::CountFloat(3.5)),
count: Some(api::greptime_proto::io::prometheus::write::v2::histogram::Count::CountFloat(6.0)),
zero_count: Some(
api::greptime_proto::io::prometheus::write::v2::histogram::ZeroCount::ZeroCountFloat(
0.5,
@@ -1107,7 +1658,7 @@ mod tests {
assert_eq!(histogram_field_value(&rows, 1, COUNT_U64_FIELD), None);
assert_eq!(
histogram_field_value(&rows, 1, COUNT_F64_FIELD),
Some(ValueData::F64Value(3.5))
Some(ValueData::F64Value(6.0))
);
assert!(is_empty_list(histogram_field_value(
&rows,
@@ -1131,6 +1682,13 @@ mod tests {
)
}
fn request_with_histogram(histogram: Histogram) -> Request {
test_util::request_with_labels_and_histograms(
vec![(METRIC_NAME_LABEL, "metric")],
vec![histogram],
)
}
fn assert_invalid(name: &str, request: Request, expected: &str) {
let err = match into_write_requests(request) {
Ok(_) => panic!("{name}: expected invalid request error"),
tests-integration/tests/http.rs
+3 -4
View File
@@ -2702,7 +2702,7 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag
],
histograms: vec![
Histogram {
count: Some(Count::CountInt(4)),
count: Some(Count::CountInt(8)),
sum: 10.0,
schema: 1,
zero_threshold: 0.001,
@@ -2720,11 +2720,10 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag
reset_hint: 2,
timestamp: 3000,
start_timestamp: 1500,
custom_values: vec![0.5, 1.5],
..Default::default()
},
Histogram {
count: Some(Count::CountFloat(3.5)),
count: Some(Count::CountFloat(6.0)),
sum: 20.0,
schema: 2,
zero_threshold: 0.002,
@@ -2773,7 +2772,7 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag
"prometheus_remote_write_v2_native_histogram_rows",
&client,
"select greptime_timestamp, greptime_native_histogram, job, instance from remote_write_v2_latency_seconds order by greptime_timestamp;",
"[[3000,{\"count_f64\":null,\"count_u64\":4,\"custom_values\":[0.5,1.5],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_u64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":3.5,\"count_u64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_u64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]",
"[[3000,{\"count_f64\":null,\"count_u64\":8,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_u64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":6.0,\"count_u64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_u64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]",
)
.await;