From 39d3744f4f40940021320c0f415ebb6afaf79d99 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Mon, 9 Feb 2026 17:51:00 +0800 Subject: [PATCH] fix: bump prometheus to 0.14 (#7686) * chore: add metrics test for fs backend Signed-off-by: evenyag * test: test opendal metrics Signed-off-by: evenyag * fix: update prometheus to 0.14 The opendal metrics require 0.14. Signed-off-by: evenyag --------- Signed-off-by: evenyag --- Cargo.lock | 73 +++++++------------- Cargo.toml | 2 +- src/common/telemetry/src/metric.rs | 74 +++++++++------------ src/object-store/tests/object_store_test.rs | 25 +++++++ src/operator/src/delete.rs | 2 +- src/servers/src/http/jaeger.rs | 10 +-- 6 files changed, 89 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c496f6879f..cfb3169a2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1693,7 +1693,7 @@ dependencies = [ "object-store", "partition", "paste", - "prometheus 0.13.4", + "prometheus 0.14.0", "promql-parser", "rand 0.9.1", "serde", @@ -2080,7 +2080,7 @@ dependencies = [ "lazy_static", "moka", "parking_lot 0.12.4", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "query", "rand 0.9.1", @@ -2180,7 +2180,7 @@ dependencies = [ "parquet", "plugins", "pprof", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "query", "rand 0.9.1", @@ -2640,7 +2640,7 @@ dependencies = [ "lazy_static", "moka", "object-store", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "prost-types 0.14.1", "rand 0.9.1", @@ -2796,7 +2796,7 @@ dependencies = [ "parking_lot 0.12.4", "paste", "pin-project", - "prometheus 0.13.4", + "prometheus 0.14.0", "ratelimit", "serde", "serde_json", @@ -2847,7 +2847,7 @@ dependencies = [ "lazy_static", "nix 0.30.1", "num_cpus", - "prometheus 0.13.4", + "prometheus 0.14.0", "sysinfo", "tokio", ] @@ -2870,7 +2870,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk 0.30.0", "parking_lot 0.12.4", - "prometheus 0.13.4", + "prometheus 0.14.0", "serde", "serde_json", "tokio", @@ -4339,7 +4339,7 @@ dependencies = [ "num_cpus", "object-store", "partition", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "query", "reqwest", @@ -5224,7 +5224,7 @@ dependencies = [ "operator", "partition", "pretty_assertions", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "query", "rand 0.9.1", @@ -5352,7 +5352,7 @@ dependencies = [ "otel-arrow-rust", "partition", "pipeline", - "prometheus 0.13.4", + "prometheus 0.14.0", "promql-parser", "prost 0.14.1", "query", @@ -7499,7 +7499,7 @@ dependencies = [ "lazy_static", "object-store", "pin-project", - "prometheus 0.13.4", + "prometheus 0.14.0", "protobuf 2.28.0", "protobuf-build", "raft-engine", @@ -7856,7 +7856,7 @@ dependencies = [ "ordered-float 4.6.0", "parking_lot 0.12.4", "partition", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "rand 0.9.1", "regex", @@ -7940,7 +7940,7 @@ dependencies = [ "mur3", "object-store", "partition", - "prometheus 0.13.4", + "prometheus 0.14.0", "serde", "serde_json", "smallvec", @@ -8082,7 +8082,7 @@ dependencies = [ "partition", "paste", "pin-project", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "puffin", "rand 0.9.1", @@ -8785,7 +8785,7 @@ dependencies = [ "lazy_static", "moka", "opendal", - "prometheus 0.13.4", + "prometheus 0.14.0", "reqwest", "serde", "snafu 0.8.6", @@ -9147,7 +9147,7 @@ dependencies = [ "object_store_opendal", "partition", "path-slash", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "prost-types 0.14.1", "query", @@ -9845,7 +9845,7 @@ dependencies = [ "operator", "ordered-float 4.6.0", "paste", - "prometheus 0.13.4", + "prometheus 0.14.0", "query", "rayon", "regex", @@ -10207,19 +10207,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "procfs" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" -dependencies = [ - "bitflags 2.9.1", - "hex", - "lazy_static", - "procfs-core 0.16.0", - "rustix 0.38.44", -] - [[package]] name = "procfs" version = "0.17.0" @@ -10228,20 +10215,10 @@ checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ "bitflags 2.9.1", "hex", - "procfs-core 0.17.0", + "procfs-core", "rustix 0.38.44", ] -[[package]] -name = "procfs-core" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" -dependencies = [ - "bitflags 2.9.1", - "hex", -] - [[package]] name = "procfs-core" version = "0.17.0" @@ -10261,10 +10238,8 @@ dependencies = [ "cfg-if", "fnv", "lazy_static", - "libc", "memchr", "parking_lot 0.12.4", - "procfs 0.16.0", "protobuf 2.28.0", "thiserror 1.0.69", ] @@ -10281,7 +10256,7 @@ dependencies = [ "libc", "memchr", "parking_lot 0.12.4", - "procfs 0.17.0", + "procfs", "protobuf 3.7.2", "thiserror 2.0.17", ] @@ -10316,7 +10291,7 @@ dependencies = [ "futures", "greptime-proto", "lazy_static", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "snafu 0.8.6", "tokio", @@ -10689,7 +10664,7 @@ dependencies = [ "lz4_flex", "moka", "pin-project", - "prometheus 0.13.4", + "prometheus 0.14.0", "serde", "serde_json", "sha2", @@ -10785,7 +10760,7 @@ dependencies = [ "partition", "paste", "pretty_assertions", - "prometheus 0.13.4", + "prometheus 0.14.0", "promql", "promql-parser", "prost 0.14.1", @@ -12284,7 +12259,7 @@ dependencies = [ "pipeline", "postgres-types", "pprof", - "prometheus 0.13.4", + "prometheus 0.14.0", "promql-parser", "prost 0.14.1", "query", @@ -13082,7 +13057,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "num_enum 0.7.4", - "prometheus 0.13.4", + "prometheus 0.14.0", "prost 0.14.1", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 9f0ab69917..ce3137df67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,7 @@ parquet = { version = "57.0", default-features = false, features = ["arrow", "as paste = "1.0" pin-project = "1.0" pretty_assertions = "1.4.0" -prometheus = { version = "0.13.3", features = ["process"] } +prometheus = { version = "0.14", features = ["process"] } promql-parser = { version = "0.7.1", features = ["ser"] } prost = { version = "0.14", features = ["no-recursion-limit"] } prost-types = "0.14" diff --git a/src/common/telemetry/src/metric.rs b/src/common/telemetry/src/metric.rs index 317b7bc3fb..e75688c2d7 100644 --- a/src/common/telemetry/src/metric.rs +++ b/src/common/telemetry/src/metric.rs @@ -55,47 +55,47 @@ pub fn convert_metric_to_write_request( if !metric_filter.map(|f| f.filter(&mf)).unwrap_or(true) { continue; } - let mf_type = mf.get_field_type(); - let mf_name = mf.get_name(); + let mf_type = mf.type_(); + let mf_name = mf.name(); for m in mf.get_metric() { - let timestamp = if m.get_timestamp_ms() == 0 { + let timestamp = if m.timestamp_ms() == 0 { default_timestamp } else { - m.get_timestamp_ms() + m.timestamp_ms() }; match mf_type { MetricType::COUNTER => timeseries.push(TimeSeries { - labels: convert_label(m.get_label(), mf_name, None), + labels: convert_label(&m.label, mf_name, None), samples: vec![Sample { - value: m.get_counter().get_value(), + value: m.counter.value(), timestamp, }], exemplars: vec![], histograms: vec![], }), MetricType::GAUGE => timeseries.push(TimeSeries { - labels: convert_label(m.get_label(), mf_name, None), + labels: convert_label(&m.label, mf_name, None), samples: vec![Sample { - value: m.get_gauge().get_value(), + value: m.gauge.value(), timestamp, }], exemplars: vec![], histograms: vec![], }), MetricType::HISTOGRAM => { - let h = m.get_histogram(); + let h = &m.histogram; let mut inf_seen = false; let metric_name = format!("{}_bucket", mf_name); - for b in h.get_bucket() { - let upper_bound = b.get_upper_bound(); + for b in &h.bucket { + let upper_bound = b.upper_bound(); timeseries.push(TimeSeries { labels: convert_label( - m.get_label(), + &m.label, metric_name.as_str(), Some(("le", upper_bound.to_string())), ), samples: vec![Sample { - value: b.get_cumulative_count() as f64, + value: b.cumulative_count() as f64, timestamp, }], exemplars: vec![], @@ -108,12 +108,12 @@ pub fn convert_metric_to_write_request( if !inf_seen { timeseries.push(TimeSeries { labels: convert_label( - m.get_label(), + &m.label, metric_name.as_str(), Some(("le", "+Inf".to_string())), ), samples: vec![Sample { - value: h.get_sample_count() as f64, + value: h.sample_count() as f64, timestamp, }], exemplars: vec![], @@ -121,13 +121,9 @@ pub fn convert_metric_to_write_request( }); } timeseries.push(TimeSeries { - labels: convert_label( - m.get_label(), - format!("{}_sum", mf_name).as_str(), - None, - ), + labels: convert_label(&m.label, format!("{}_sum", mf_name).as_str(), None), samples: vec![Sample { - value: h.get_sample_sum(), + value: h.sample_sum(), timestamp, }], exemplars: vec![], @@ -135,12 +131,12 @@ pub fn convert_metric_to_write_request( }); timeseries.push(TimeSeries { labels: convert_label( - m.get_label(), + &m.label, format!("{}_count", mf_name).as_str(), None, ), samples: vec![Sample { - value: h.get_sample_count() as f64, + value: h.sample_count() as f64, timestamp, }], exemplars: vec![], @@ -148,16 +144,16 @@ pub fn convert_metric_to_write_request( }); } MetricType::SUMMARY => { - let s = m.get_summary(); - for q in s.get_quantile() { + let s = &m.summary; + for q in &s.quantile { timeseries.push(TimeSeries { labels: convert_label( - m.get_label(), + &m.label, mf_name, - Some(("quantile", q.get_quantile().to_string())), + Some(("quantile", q.quantile().to_string())), ), samples: vec![Sample { - value: q.get_value(), + value: q.value(), timestamp, }], exemplars: vec![], @@ -165,13 +161,9 @@ pub fn convert_metric_to_write_request( }); } timeseries.push(TimeSeries { - labels: convert_label( - m.get_label(), - format!("{}_sum", mf_name).as_str(), - None, - ), + labels: convert_label(&m.label, format!("{}_sum", mf_name).as_str(), None), samples: vec![Sample { - value: s.get_sample_sum(), + value: s.sample_sum(), timestamp, }], exemplars: vec![], @@ -179,12 +171,12 @@ pub fn convert_metric_to_write_request( }); timeseries.push(TimeSeries { labels: convert_label( - m.get_label(), + &m.label, format!("{}_count", mf_name).as_str(), None, ), samples: vec![Sample { - value: s.get_sample_count() as f64, + value: s.sample_count() as f64, timestamp, }], exemplars: vec![], @@ -212,8 +204,8 @@ fn convert_label( let mut labels = Vec::with_capacity(pairs.len() + 1 + if addon.is_some() { 1 } else { 0 }); for label in pairs { labels.push(remote::Label { - name: label.get_name().to_string(), - value: label.get_value().to_string(), + name: label.name().to_string(), + value: label.value().to_string(), }); } labels.push(remote::Label { @@ -352,11 +344,11 @@ TimeSeries { labels: [Label { name: "__name__", value: "test_histogram_count" }, quantile2.set_quantile(100.0); quantile2.set_value(5.0); - summary.set_quantile(vec![quantile1, quantile2].into()); + summary.set_quantile(vec![quantile1, quantile2]); let mut metric = Metric::default(); metric.set_summary(summary); - metric_family.set_metric(vec![metric].into()); + metric_family.set_metric(vec![metric]); let write_quest = convert_metric_to_write_request(vec![metric_family], None, 20); let write_quest_str: Vec<_> = write_quest @@ -388,7 +380,7 @@ TimeSeries { labels: [Label { name: "__name__", value: "test_summary_count" }], mf.append(&mut counter_2.collect()); let filter = MetricFilter::new(Arc::new(|mf: &MetricFamily| { - !mf.get_name().starts_with("filter") + !mf.name().starts_with("filter") })); let write_quest1 = convert_metric_to_write_request(mf.clone(), None, 0); let write_quest2 = convert_metric_to_write_request(mf, Some(&filter), 0); diff --git a/src/object-store/tests/object_store_test.rs b/src/object-store/tests/object_store_test.rs index 4a8730111c..5df6ab04bd 100644 --- a/src/object-store/tests/object_store_test.rs +++ b/src/object-store/tests/object_store_test.rs @@ -22,6 +22,7 @@ use object_store::services::{Fs, S3}; use object_store::test_util::TempFolder; use opendal::EntryMode; use opendal::services::{Azblob, Gcs, Oss}; +use prometheus::{Encoder, TextEncoder}; async fn test_object_crud(store: &ObjectStore) -> Result<()> { // Create object handler. @@ -102,6 +103,19 @@ async fn test_object_list(store: &ObjectStore) -> Result<()> { Ok(()) } +fn assert_opendal_metrics() { + let metric_families = prometheus::gather(); + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + let text = String::from_utf8(buffer).unwrap(); + assert!( + text.contains("opendal"), + "Missing opendal metrics: {}", + text + ); +} + #[tokio::test] async fn test_fs_backend() -> Result<()> { let data_dir = create_temp_dir("test_fs_backend"); @@ -111,10 +125,13 @@ async fn test_fs_backend() -> Result<()> { .atomic_write_dir(&tmp_dir.path().to_string_lossy()); let store = ObjectStore::new(builder).unwrap().finish(); + let store = object_store::util::with_instrument_layers(store, false); test_object_crud(&store).await?; test_object_list(&store).await?; + assert_opendal_metrics(); + Ok(()) } @@ -136,10 +153,12 @@ async fn test_s3_backend() -> Result<()> { .bucket(&bucket); let store = ObjectStore::new(builder).unwrap().finish(); + let store = object_store::util::with_instrument_layers(store, false); let guard = TempFolder::new(&store, "/"); test_object_crud(&store).await?; test_object_list(&store).await?; + assert_opendal_metrics(); guard.remove_all().await?; } @@ -163,10 +182,12 @@ async fn test_oss_backend() -> Result<()> { .bucket(&bucket); let store = ObjectStore::new(builder).unwrap().finish(); + let store = object_store::util::with_instrument_layers(store, false); let guard = TempFolder::new(&store, "/"); test_object_crud(&store).await?; test_object_list(&store).await?; + assert_opendal_metrics(); guard.remove_all().await?; } @@ -190,10 +211,12 @@ async fn test_azblob_backend() -> Result<()> { .container(&container); let store = ObjectStore::new(builder).unwrap().finish(); + let store = object_store::util::with_instrument_layers(store, false); let guard = TempFolder::new(&store, "/"); test_object_crud(&store).await?; test_object_list(&store).await?; + assert_opendal_metrics(); guard.remove_all().await?; } Ok(()) @@ -216,10 +239,12 @@ async fn test_gcs_backend() -> Result<()> { .endpoint(&env::var("GT_GCS_ENDPOINT").unwrap()); let store = ObjectStore::new(builder).unwrap().finish(); + let store = object_store::util::with_instrument_layers(store, false); let guard = TempFolder::new(&store, "/"); test_object_crud(&store).await?; test_object_list(&store).await?; + assert_opendal_metrics(); guard.remove_all().await?; } Ok(()) diff --git a/src/operator/src/delete.rs b/src/operator/src/delete.rs index 1e9fef919a..9641e0e27d 100644 --- a/src/operator/src/delete.rs +++ b/src/operator/src/delete.rs @@ -152,7 +152,7 @@ impl Deleter { .map(|resp| resp.map(|r| r.affected_rows)) .sum::>()?; crate::metrics::DIST_DELETE_ROW_COUNT - .with_label_values(&[ctx.get_db_string().as_ref()]) + .with_label_values(&[ctx.get_db_string().as_str()]) .inc_by(affected_rows as u64); Ok(affected_rows) } diff --git a/src/servers/src/http/jaeger.rs b/src/servers/src/http/jaeger.rs index 148e2ac77a..c729f6d85d 100644 --- a/src/servers/src/http/jaeger.rs +++ b/src/servers/src/http/jaeger.rs @@ -411,7 +411,7 @@ pub async fn handle_get_services( // Record the query time histogram. let _timer = METRIC_JAEGER_QUERY_ELAPSED - .with_label_values(&[&db, "/api/services"]) + .with_label_values(&[db.as_str(), "/api/services"]) .start_timer(); match handler.get_services(query_ctx).await { @@ -464,7 +464,7 @@ pub async fn handle_get_trace( // Record the query time histogram. let _timer = METRIC_JAEGER_QUERY_ELAPSED - .with_label_values(&[&db, "/api/traces"]) + .with_label_values(&[db.as_str(), "/api/traces"]) .start_timer(); // Convert start time and end time from microseconds to nanoseconds. @@ -541,7 +541,7 @@ pub async fn handle_find_traces( // Record the query time histogram. let _timer = METRIC_JAEGER_QUERY_ELAPSED - .with_label_values(&[&db, "/api/traces"]) + .with_label_values(&[db.as_str(), "/api/traces"]) .start_timer(); match QueryTraceParams::from_jaeger_query_params(query_params) { @@ -597,7 +597,7 @@ pub async fn handle_get_operations( // Record the query time histogram. let _timer = METRIC_JAEGER_QUERY_ELAPSED - .with_label_values(&[&db, "/api/operations"]) + .with_label_values(&[db.as_str(), "/api/operations"]) .start_timer(); match handler @@ -671,7 +671,7 @@ pub async fn handle_get_operations_by_service( // Record the query time histogram. let _timer = METRIC_JAEGER_QUERY_ELAPSED - .with_label_values(&[&db, "/api/services"]) + .with_label_values(&[db.as_str(), "/api/services"]) .start_timer(); match handler.get_operations(query_ctx, &service_name, None).await {