From 21aed0371f61b2f0b2ed73aef42dea689e789351 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Fri, 11 Sep 2026 09:39:13 +0000 Subject: [PATCH] feat(pprof): switch CPU profiler to framehop unwinder (#9125) * feat(pprof): switch CPU profiler to framehop unwinder Replace the default libgcc-based unwinder in pprof-rs with the framehop unwinder, which is designed to be async-signal-safe: - framehop performs no heap allocation during unwinding (MustNotAllocateDuringUnwind) - It handles prologue/epilogue interruption correctly - It falls back to frame-pointer unwinding when CFI is unavailable - It does not depend on libgcc's unwind implementation, which is documented as not signal-safe (see tikv/pprof-rs#36) Bump pprof from 0.14 to 0.15 in all three consumers (common-pprof, cmd, servers) to unify on a single version. pprof 0.15 also replaces parking_lot with spin-rs to avoid a potential profiler deadlock (#268). This addresses the libgcc_s.so.1 #GP crash observed in production when CPU profiling is active, by eliminating the libgcc unwinder from the signal handler path entirely. Signed-off-by: Lei, HUANG * fix(pprof): gate framehop-unwinder to supported targets framehop-unwinder is only available on x86_64/aarch64 Linux/macOS. Enabling it unconditionally for all Unix targets breaks the build on riscv64 and other platforms: pprof disables its backtrace-rs fallback when framehop-unwinder is set, but the framehop module is not compiled on unsupported targets, leaving no TraceImpl implementation. Split the pprof dependency: the base target.'cfg(unix)' block carries the common features (flamegraph, prost-codec, protobuf), and a separate target block adds framehop-unwinder only on supported targets. Cargo unions features from both blocks on matching targets, so x86_64 and aarch64 Linux/macOS get the full feature set while other Unix targets fall back to the default backtrace-rs implementation. Addresses review comment discussion_r3987313632. Signed-off-by: Lei, HUANG --------- Signed-off-by: Lei, HUANG --- Cargo.lock | 123 ++++++++++++++++++++++++++++-------- src/cmd/Cargo.toml | 2 +- src/common/pprof/Cargo.toml | 8 ++- src/servers/Cargo.toml | 2 +- 4 files changed, 107 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40f7cba32f..3876eb6d7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1172,7 +1172,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.36.7", "rustc-demangle", "windows-targets 0.52.6", ] @@ -2271,7 +2271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -5177,7 +5177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -5265,6 +5265,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fallible-streaming-iterator" version = "0.1.9" @@ -5601,6 +5607,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" +[[package]] +name = "framehop" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe" +dependencies = [ + "arrayvec", + "cfg-if", + "fallible-iterator 0.3.0", + "gimli", + "macho-unwind-info", + "pe-unwind-info", +] + [[package]] name = "frontend" version = "1.3.0-alpha.1" @@ -5746,7 +5766,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes 2.0.4", "rustix 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6062,6 +6082,10 @@ name = "gimli" version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "fallible-iterator 0.3.0", + "stable_deref_trait", +] [[package]] name = "git2" @@ -7143,7 +7167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" dependencies = [ "io-lifetimes 3.0.1", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -7181,7 +7205,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -7884,7 +7908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.5", + "windows-targets 0.48.5", ] [[package]] @@ -8230,6 +8254,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "macho-unwind-info" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149" +dependencies = [ + "thiserror 2.0.17", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "mappings" version = "0.7.1" @@ -8346,6 +8381,15 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memmap2" version = "0.9.5" @@ -9443,6 +9487,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "flate2", + "memchr", +] + [[package]] name = "object" version = "0.36.7" @@ -10484,6 +10538,19 @@ dependencies = [ "hmac 0.12.1", ] +[[package]] +name = "pe-unwind-info" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59" +dependencies = [ + "arrayvec", + "bitflags 2.12.1", + "thiserror 2.0.17", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "peeking_take_while" version = "1.0.0" @@ -10997,7 +11064,7 @@ dependencies = [ "base64 0.22.1", "byteorder", "bytes", - "fallible-iterator", + "fallible-iterator 0.2.0", "hmac 0.13.0", "md-5 0.11.0", "memchr", @@ -11015,7 +11082,7 @@ dependencies = [ "array-init", "bytes", "chrono", - "fallible-iterator", + "fallible-iterator 0.2.0", "postgres-protocol", "serde_core", "serde_json", @@ -11038,30 +11105,33 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebbe2f8898beba44815fdc9e5a4ae9c929e21c5dc29b0c774a15555f7f58d6d0" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" dependencies = [ "aligned-vec", "backtrace", "cfg-if", "criterion 0.5.1", "findshlibs", + "framehop", "inferno 0.11.21", "libc", "log", + "memmap2 0.5.10", "nix 0.26.4", + "object 0.29.0", "once_cell", - "parking_lot 0.12.4", "prost 0.12.6", "prost-build 0.12.6", "prost-derive 0.12.6", - "protobuf 2.28.0", + "protobuf 3.7.2", "sha2 0.10.9", "smallvec", + "spin 0.10.1", "symbolic-demangle", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.17", ] [[package]] @@ -11922,7 +11992,7 @@ dependencies = [ "once_cell", "socket2 0.5.10", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -12893,7 +12963,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -12906,7 +12976,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -12997,7 +13067,7 @@ dependencies = [ "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -13953,6 +14023,9 @@ name = "spin" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -14304,7 +14377,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -14583,7 +14656,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a1150bdda9314f6cfeeea801c23f5593c6e6a6c72e64f67e48d723a12b8efdb" dependencies = [ "debugid", - "memmap2", + "memmap2 0.9.5", "stable_deref_trait", "uuid", ] @@ -14752,7 +14825,7 @@ dependencies = [ "lru 0.16.4", "lz4_flex 0.13.1", "measure_time", - "memmap2", + "memmap2 0.9.5", "once_cell", "oneshot", "rayon", @@ -14921,7 +14994,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.0.7", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -15379,7 +15452,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "fallible-iterator", + "fallible-iterator 0.2.0", "futures-channel", "futures-util", "log", @@ -16728,7 +16801,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -17178,7 +17251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ "bitflags 2.12.1", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index 9c7ca6adea..722e48ae90 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -119,7 +119,7 @@ tracing-appender.workspace = true [target.'cfg(unix)'.dependencies] daemonize = "0.5" -pprof = { version = "0.14", features = [ +pprof = { version = "0.15", features = [ "flamegraph", ] } diff --git a/src/common/pprof/Cargo.toml b/src/common/pprof/Cargo.toml index 6d96b26d9f..e3a5bce78e 100644 --- a/src/common/pprof/Cargo.toml +++ b/src/common/pprof/Cargo.toml @@ -11,11 +11,17 @@ snafu.workspace = true tokio.workspace = true [target.'cfg(unix)'.dependencies] -pprof = { version = "0.14", features = [ +pprof = { version = "0.15", features = [ "flamegraph", "prost-codec", "protobuf", ] } +# framehop-unwinder is only available on x86_64/aarch64 Linux/macOS. +# Enabling it on other targets (e.g. riscv64) would disable the +# backtrace-rs fallback without providing a TraceImpl, breaking the build. +[target.'cfg(all(unix, any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos")))'.dependencies] +pprof = { version = "0.15", features = ["framehop-unwinder"] } + [lints] workspace = true diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index 7975d02e5b..dbb956c0d5 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -172,7 +172,7 @@ tokio-postgres.workspace = true tokio-postgres-rustls = "0.14" [target.'cfg(unix)'.dev-dependencies] -pprof = { version = "0.14", features = ["criterion", "flamegraph"] } +pprof = { version = "0.15", features = ["criterion", "flamegraph"] } [target.'cfg(not(windows))'.dev-dependencies] tikv-jemallocator = "0.7"