diff --git a/Cargo.lock b/Cargo.lock index df06ede090..6770b00b97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,7 +553,7 @@ dependencies = [ "log", "parking", "polling", - "rustix", + "rustix 0.37.14", "slab", "socket2 0.4.9", "waker-fn", @@ -842,6 +842,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", +] + [[package]] name = "bindgen" version = "0.65.1" @@ -1202,7 +1222,7 @@ dependencies = [ "lazy_static", "log-store", "meta-client", - "metrics", + "metrics 0.21.0", "mito", "moka 0.11.0", "object-store", @@ -1840,7 +1860,7 @@ dependencies = [ "async-trait", "common-error", "common-telemetry", - "metrics", + "metrics 0.21.0", "once_cell", "paste", "snafu", @@ -1856,7 +1876,7 @@ dependencies = [ "backtrace", "common-error", "console-subscriber", - "metrics", + "metrics 0.21.0", "metrics-exporter-prometheus", "metrics-util", "once_cell", @@ -2557,7 +2577,7 @@ dependencies = [ "log-store", "meta-client", "meta-srv", - "metrics", + "metrics 0.21.0", "mito", "object-store", "pin-project", @@ -2879,6 +2899,17 @@ dependencies = [ "serde", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + [[package]] name = "errno" version = "0.3.1" @@ -2980,7 +3011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ae6b3d9530211fb3b12a95374b8b0823be812f53d09e18c5675c0146b09642" dependencies = [ "cfg-if 1.0.0", - "rustix", + "rustix 0.37.14", "windows-sys 0.48.0", ] @@ -3118,7 +3149,7 @@ dependencies = [ "meta-srv", "meter-core", "meter-macros", - "metrics", + "metrics 0.21.0", "mito", "moka 0.9.7", "object-store", @@ -4232,7 +4263,7 @@ checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", "number_prefix", - "portable-atomic", + "portable-atomic 0.3.19", "unicode-width", ] @@ -4339,7 +4370,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix", + "rustix 0.37.14", "windows-sys 0.48.0", ] @@ -4561,6 +4592,17 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +[[package]] +name = "libproc" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b18cbf29f8ff3542ba22bdce9ac610fcb75d74bb4e2b306b2a2762242025b4f" +dependencies = [ + "bindgen 0.64.0", + "errno 0.2.8", + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.25.2" @@ -4598,6 +4640,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "linux-raw-sys" version = "0.3.4" @@ -4940,7 +4988,7 @@ dependencies = [ "h2", "http-body", "lazy_static", - "metrics", + "metrics 0.21.0", "once_cell", "parking_lot", "prost", @@ -4986,22 +5034,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ "ahash 0.7.6", - "metrics-macros", - "portable-atomic", + "metrics-macros 0.6.0", + "portable-atomic 0.3.19", +] + +[[package]] +name = "metrics" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8ebbd1a9e57bbab77b9facae7f5136aea44c356943bf9a198f647da64285d6" +dependencies = [ + "ahash 0.8.3", + "metrics-macros 0.7.0", + "portable-atomic 1.3.2", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" +checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ + "base64 0.21.0", "indexmap", - "metrics", + "metrics 0.21.0", "metrics-util", - "parking_lot", - "portable-atomic", - "quanta 0.10.1", + "quanta 0.11.0", "thiserror", ] @@ -5017,22 +5075,46 @@ dependencies = [ ] [[package]] -name = "metrics-util" -version = "0.14.0" +name = "metrics-macros" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "metrics-process" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e7caea6cb64b66f96f7ab0e5f8bd8334d72dbbc522c99bed71d8a382d515f" +dependencies = [ + "libproc", + "mach2", + "metrics 0.21.0", + "once_cell", + "procfs", + "rlimit", + "windows 0.48.0", +] + +[[package]] +name = "metrics-util" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" dependencies = [ "aho-corasick 0.7.20", "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.12.3", + "hashbrown 0.13.2", "indexmap", - "metrics", + "metrics 0.21.0", "num_cpus", - "ordered-float 2.10.0", - "parking_lot", - "portable-atomic", - "quanta 0.10.1", + "ordered-float 3.7.0", + "quanta 0.11.0", "radix_trie", "sketches-ddsketch", ] @@ -5244,7 +5326,7 @@ checksum = "e6b76684cc6825e9e5f3d9d41968faf04c6f9eb39815dc9827695b1eb5faa826" dependencies = [ "base64 0.21.0", "bigdecimal", - "bindgen", + "bindgen 0.65.1", "bitflags 2.1.0", "bitvec", "byteorder", @@ -5552,7 +5634,7 @@ dependencies = [ "futures", "lru 0.9.0", "md5", - "metrics", + "metrics 0.21.0", "opendal", "pin-project", "tokio", @@ -5610,7 +5692,7 @@ dependencies = [ "hyper", "log", "md-5", - "metrics", + "metrics 0.20.1", "once_cell", "parking_lot", "percent-encoding", @@ -6216,6 +6298,12 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" +[[package]] +name = "portable-atomic" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc59d1bcc64fc5d021d67521f818db868368028108d37f0e98d74e33f68297b5" + [[package]] name = "postgres-protocol" version = "0.6.5" @@ -6356,6 +6444,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "hex", + "lazy_static", + "rustix 0.36.14", +] + [[package]] name = "prodash" version = "23.1.2" @@ -6669,7 +6770,7 @@ dependencies = [ "futures", "futures-util", "humantime", - "metrics", + "metrics 0.21.0", "num", "num-traits", "object-store", @@ -7078,6 +7179,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rlimit" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a29d87a652dc4d43c586328706bb5cdff211f3f39a530f240b53f7221dab8e" +dependencies = [ + "libc", +] + [[package]] name = "ron" version = "0.7.1" @@ -7240,6 +7350,20 @@ dependencies = [ "semver 1.0.17", ] +[[package]] +name = "rustix" +version = "0.36.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" +dependencies = [ + "bitflags 1.3.2", + "errno 0.3.1", + "io-lifetimes", + "libc", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", +] + [[package]] name = "rustix" version = "0.37.14" @@ -7247,10 +7371,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f" dependencies = [ "bitflags 1.3.2", - "errno", + "errno 0.3.1", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.4", "windows-sys 0.48.0", ] @@ -8062,7 +8186,8 @@ dependencies = [ "humantime-serde", "hyper", "influxdb_line_protocol", - "metrics", + "metrics 0.21.0", + "metrics-process", "mime_guess", "mysql_async", "num_cpus", @@ -8521,7 +8646,7 @@ dependencies = [ "futures-util", "lazy_static", "log-store", - "metrics", + "metrics 0.21.0", "object-store", "parquet", "paste", @@ -8857,7 +8982,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", - "rustix", + "rustix 0.37.14", "windows-sys 0.45.0", ] diff --git a/Cargo.toml b/Cargo.toml index 1cd431c9c5..6a67f5b804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,7 +84,7 @@ tokio = { version = "1.28", features = ["full"] } tokio-util = { version = "0.7", features = ["io-util", "compat"] } tonic = { version = "0.9", features = ["tls"] } uuid = { version = "1", features = ["serde", "v4", "fast-rng"] } -metrics = "0.20" +metrics = "0.21" meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "f0798c4c648d89f51abe63e870919c75dd463199" } [workspace.dependencies.meter-macros] diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml index f67cffa685..9d7ebe817f 100644 --- a/src/common/telemetry/Cargo.toml +++ b/src/common/telemetry/Cargo.toml @@ -12,8 +12,8 @@ deadlock_detection = ["parking_lot"] backtrace = "0.3" common-error = { path = "../error" } console-subscriber = { version = "0.1", optional = true } -metrics-exporter-prometheus = { version = "0.11", default-features = false } -metrics-util = "0.14.0" +metrics-exporter-prometheus = { version = "0.12", default-features = false } +metrics-util = "0.15" metrics.workspace = true once_cell = "1.10" opentelemetry = { version = "0.17", default-features = false, features = [ diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 19806c1769..47e44f1ae8 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -10,7 +10,7 @@ async-trait = "0.1" bytes = "1.4" futures = { version = "0.3" } md5 = "0.7" -metrics = "0.20" +metrics.workspace = true opendal = { version = "0.33", features = ["layers-tracing", "layers-metrics"] } pin-project = "1.0" tokio.workspace = true diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index 47caff1efe..8ff51bb536 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -40,6 +40,7 @@ humantime-serde = "1.1" hyper = { version = "0.14", features = ["full"] } influxdb_line_protocol = { git = "https://github.com/evenyag/influxdb_iox", branch = "feat/line-protocol" } metrics.workspace = true +metrics-process = "1" mime_guess = "2.0" num_cpus = "1.13" once_cell = "1.16" diff --git a/src/servers/src/http/handler.rs b/src/servers/src/http/handler.rs index 645b4110e9..70b9cefba1 100644 --- a/src/servers/src/http/handler.rs +++ b/src/servers/src/http/handler.rs @@ -26,6 +26,7 @@ use serde::{Deserialize, Serialize}; use session::context::UserInfo; use crate::http::{ApiState, JsonResponse}; +use crate::metrics::PROCESS_COLLECTOR; use crate::metrics_handler::MetricsHandler; #[derive(Debug, Default, Serialize, Deserialize, JsonSchema)] @@ -128,6 +129,9 @@ pub async fn metrics( State(state): State, Query(_params): Query>, ) -> String { + // Collect process metrics. + PROCESS_COLLECTOR.collect(); + state.render() } diff --git a/src/servers/src/metrics.rs b/src/servers/src/metrics.rs index 69f26f3a77..b0f171e5c5 100644 --- a/src/servers/src/metrics.rs +++ b/src/servers/src/metrics.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use metrics_process::Collector; +use once_cell::sync::Lazy; + pub(crate) const METRIC_DB_LABEL: &str = "db"; pub(crate) const METRIC_CODE_LABEL: &str = "code"; @@ -47,3 +50,11 @@ pub(crate) const METRIC_HTTP_REQUESTS_ELAPSED: &str = "servers.http_requests_ela pub(crate) const METRIC_METHOD_LABEL: &str = "method"; pub(crate) const METRIC_PATH_LABEL: &str = "path"; pub(crate) const METRIC_STATUS_LABEL: &str = "status"; + +/// Prometheus style process metrics collector. +pub(crate) static PROCESS_COLLECTOR: Lazy = Lazy::new(|| { + let collector = Collector::default(); + // Describe collector. + collector.describe(); + collector +});