Add a test for generating pprof from collapsed

This provides the binaries and libraries required for the continuous
profiling at runtime.

Cargo.lock

@@ -687,13 +687,40 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "axum"
+version = "0.7.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
+dependencies = [
+ "async-trait",
+ "axum-core 0.4.5",
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.0",
+ "http-body-util",
+ "itoa",
+ "matchit 0.7.3",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper 1.0.1",
+ "tower 0.5.2",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "axum"
 version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8"
 dependencies = [
- "axum-core",
+ "axum-core 0.5.0",
  "base64 0.22.1",
  "bytes",
  "form_urlencoded",
@@ -704,7 +731,7 @@ dependencies = [
  "hyper 1.4.1",
  "hyper-util",
  "itoa",
- "matchit",
+ "matchit 0.8.4",
  "memchr",
  "mime",
  "percent-encoding",
@@ -724,6 +751,26 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "axum-core"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.0",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper 1.0.1",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "axum-core"
 version = "0.5.0"
@@ -750,8 +797,8 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b"
 dependencies = [
- "axum",
- "axum-core",
+ "axum 0.8.1",
+ "axum-core 0.5.0",
  "bytes",
  "form_urlencoded",
  "futures-util",
@@ -962,6 +1009,24 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
+dependencies = [
+ "bitflags 2.8.0",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.12.1",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash 1.1.0",
+ "shlex",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "bindgen"
 version = "0.71.1"
@@ -1009,12 +1074,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "boxcar"
-version = "0.2.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26c4925bc979b677330a8c7fe7a8c94af2dbb4a2d37b4a20a80d884400f46baa"
-
 [[package]]
 name = "bstr"
 version = "1.5.0"
@@ -1248,14 +1307,14 @@ checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
 
 [[package]]
 name = "clashmap"
-version = "1.2.0"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a055b1f1bf558eae4959f6dd77cf2d7d50ae1483928e60ef21ca5a24fd4321"
+checksum = "93bd59c81e2bd87a775ae2de75f070f7e2bfe97363a6ad652f46824564c23e4d"
 dependencies = [
  "crossbeam-utils",
  "hashbrown 0.15.2",
  "lock_api",
- "parking_lot_core 0.9.10",
+ "parking_lot_core 0.9.8",
  "polonius-the-crab",
  "replace_with",
 ]
@@ -1327,7 +1386,7 @@ dependencies = [
  "aws-sdk-kms",
  "aws-sdk-s3",
  "aws-smithy-types",
- "axum",
+ "axum 0.8.1",
  "axum-extra",
  "base64 0.22.1",
  "bytes",
@@ -1342,8 +1401,10 @@ dependencies = [
  "hostname-validator",
  "http 1.1.0",
  "indexmap 2.9.0",
+ "inferno 0.12.0",
  "itertools 0.10.5",
  "jsonwebtoken",
+ "libproc",
  "metrics",
  "nix 0.30.1",
  "notify",
@@ -1357,6 +1418,8 @@ dependencies = [
  "postgres-types",
  "postgres_initdb",
  "postgres_versioninfo",
+ "pprof 0.15.0",
+ "prost 0.12.6",
  "regex",
  "remote_storage",
  "reqwest",
@@ -1368,6 +1431,7 @@ dependencies = [
  "serde_with",
  "signal-hook",
  "tar",
+ "tempfile",
  "thiserror 1.0.69",
  "tokio",
  "tokio-postgres",
@@ -1759,6 +1823,19 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "dashmap"
+version = "5.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d"
+dependencies = [
+ "cfg-if",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core 0.9.8",
+]
+
 [[package]]
 name = "dashmap"
 version = "6.1.0"
@@ -1770,7 +1847,7 @@ dependencies = [
  "hashbrown 0.14.5",
  "lock_api",
  "once_cell",
- "parking_lot_core 0.9.10",
+ "parking_lot_core 0.9.8",
 ]
 
 [[package]]
@@ -2075,7 +2152,7 @@ name = "endpoint_storage"
 version = "0.0.1"
 dependencies = [
  "anyhow",
- "axum",
+ "axum 0.8.1",
  "axum-extra",
  "camino",
  "camino-tempfile",
@@ -2191,12 +2268,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
-version = "0.3.8"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
+checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2336,12 +2413,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
-[[package]]
-name = "foldhash"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -2532,6 +2603,18 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasi 0.14.2+wasi-0.2.4",
+]
+
 [[package]]
 name = "gettid"
 version = "0.1.3"
@@ -2581,7 +2664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "842dc78579ce01e6a1576ad896edc92fca002dd60c9c3746b7fc2bec6fb429d0"
 dependencies = [
  "cfg-if",
- "dashmap",
+ "dashmap 6.1.0",
  "futures-sink",
  "futures-timer",
  "futures-util",
@@ -2789,6 +2872,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "home"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "hostname"
 version = "0.4.0"
@@ -2898,7 +2990,7 @@ dependencies = [
  "jsonwebtoken",
  "metrics",
  "once_cell",
- "pprof",
+ "pprof 0.14.0",
  "regex",
  "routerify",
  "rustls 0.23.27",
@@ -3287,7 +3379,7 @@ dependencies = [
  "clap",
  "crossbeam-channel",
  "crossbeam-utils",
- "dashmap",
+ "dashmap 6.1.0",
  "env_logger",
  "indexmap 2.9.0",
  "itoa",
@@ -3544,13 +3636,23 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "lasso"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2"
+dependencies = [
+ "dashmap 5.5.0",
+ "hashbrown 0.13.2",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 dependencies = [
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -3575,6 +3677,17 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
 
+[[package]]
+name = "libproc"
+version = "0.14.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb"
+dependencies = [
+ "bindgen 0.70.1",
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
@@ -3587,6 +3700,12 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0b5399f6804fbab912acbd8878ed3532d506b7c951b8f9f164ef90fef39e3f4"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
+
 [[package]]
 name = "litemap"
 version = "0.7.4"
@@ -3640,6 +3759,12 @@ dependencies = [
  "regex-automata 0.1.10",
 ]
 
+[[package]]
+name = "matchit"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
+
 [[package]]
 name = "matchit"
 version = "0.8.4"
@@ -3663,17 +3788,17 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "measured"
-version = "0.0.23"
+version = "0.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22ae866c28b9c59afaeb488ad6e3bd148570cf5a2bacf6c4845def5b9a03470"
+checksum = "3051f3a030d55d680cdef6ca50e80abd1182f8da29f2344a7c9cb575721138f0"
 dependencies = [
  "bytes",
  "crossbeam-utils",
  "hashbrown 0.14.5",
  "itoa",
+ "lasso",
  "measured-derive",
  "memchr",
- "paracord",
  "parking_lot 0.12.1",
  "rustc-hash 1.1.0",
  "ryu",
@@ -3681,9 +3806,9 @@ dependencies = [
 
 [[package]]
 name = "measured-derive"
-version = "0.0.23"
+version = "0.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16d734ed9dbca22d87d56b54d220f254ce921cb5cce97d4a960075af0131d076"
+checksum = "b9e6777fc80a575f9503d908c8b498782a6c3ee88a06cb416dc3941401e43b94"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -3693,26 +3818,15 @@ dependencies = [
 
 [[package]]
 name = "measured-process"
-version = "0.0.23"
+version = "0.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f71a318d2b9edcded1e5e0ccf3c9e1e1614217d7f07933631c771daa717743aa"
+checksum = "7c4b80445aeb08e832d87bf1830049a924cdc1d6b7ef40b6b9b365bff17bf8ec"
 dependencies = [
  "libc",
  "measured",
  "procfs",
 ]
 
-[[package]]
-name = "measured-tokio"
-version = "0.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e79a936051c484268e2d71a5dd01219096c62c8ff09afad95f07c14a3b0c580"
-dependencies = [
- "itoa",
- "measured",
- "tokio",
-]
-
 [[package]]
 name = "memchr"
 version = "2.6.4"
@@ -4390,7 +4504,7 @@ dependencies = [
  "postgres_ffi_types",
  "postgres_initdb",
  "posthog_client_lite",
- "pprof",
+ "pprof 0.14.0",
  "pq_proto",
  "procfs",
  "rand 0.8.5",
@@ -4571,21 +4685,6 @@ dependencies = [
  "seize",
 ]
 
-[[package]]
-name = "paracord"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e41e924113f9a05eecd561c6e695648f243a03b3ad8d9b3eff689342b95023b"
-dependencies = [
- "boxcar",
- "clashmap",
- "foldhash",
- "hashbrown 0.15.2",
- "serde",
- "sync_wrapper 1.0.1",
- "typed-arena",
-]
-
 [[package]]
 name = "parking"
 version = "2.1.1"
@@ -4610,7 +4709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.10",
+ "parking_lot_core 0.9.8",
 ]
 
 [[package]]
@@ -4629,15 +4728,15 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.10"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.5.10",
+ "redox_syscall 0.3.5",
  "smallvec",
- "windows-targets 0.52.6",
+ "windows-targets 0.48.0",
 ]
 
 [[package]]
@@ -4980,7 +5079,7 @@ name = "postgres_ffi"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "bindgen",
+ "bindgen 0.71.1",
  "bytes",
  "crc32c",
  "criterion",
@@ -4991,7 +5090,7 @@ dependencies = [
  "postgres",
  "postgres_ffi_types",
  "postgres_versioninfo",
- "pprof",
+ "pprof 0.14.0",
  "regex",
  "serde",
  "thiserror 1.0.69",
@@ -5081,6 +5180,30 @@ dependencies = [
  "thiserror 1.0.69",
 ]
 
+[[package]]
+name = "pprof"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187"
+dependencies = [
+ "aligned-vec",
+ "backtrace",
+ "cfg-if",
+ "findshlibs",
+ "inferno 0.11.21",
+ "libc",
+ "log",
+ "nix 0.26.4",
+ "once_cell",
+ "protobuf",
+ "protobuf-codegen",
+ "smallvec",
+ "spin 0.10.0",
+ "symbolic-demangle",
+ "tempfile",
+ "thiserror 2.0.11",
+]
+
 [[package]]
 name = "pprof_util"
 version = "0.7.0"
@@ -5156,7 +5279,7 @@ dependencies = [
  "hex",
  "lazy_static",
  "procfs-core",
- "rustix",
+ "rustix 0.38.41",
 ]
 
 [[package]]
@@ -5292,6 +5415,57 @@ dependencies = [
  "prost 0.13.5",
 ]
 
+[[package]]
+name = "protobuf"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
+dependencies = [
+ "once_cell",
+ "protobuf-support",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "protobuf-codegen"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace"
+dependencies = [
+ "anyhow",
+ "once_cell",
+ "protobuf",
+ "protobuf-parse",
+ "regex",
+ "tempfile",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "protobuf-parse"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973"
+dependencies = [
+ "anyhow",
+ "indexmap 2.9.0",
+ "log",
+ "protobuf",
+ "protobuf-support",
+ "tempfile",
+ "thiserror 1.0.69",
+ "which",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
+dependencies = [
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "proxy"
 version = "0.1.0"
@@ -5343,14 +5517,13 @@ dependencies = [
  "itoa",
  "jose-jwa",
  "jose-jwk",
+ "lasso",
  "measured",
- "measured-tokio",
  "metrics",
  "once_cell",
  "opentelemetry",
  "p256 0.13.2",
  "papaya",
- "paracord",
  "parking_lot 0.12.1",
  "parquet",
  "parquet_derive",
@@ -5464,6 +5637,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
 [[package]]
 name = "rand"
 version = "0.7.3"
@@ -5653,15 +5832,6 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.5.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
-dependencies = [
- "bitflags 2.8.0",
-]
-
 [[package]]
 name = "regex"
 version = "1.10.2"
@@ -5861,7 +6031,7 @@ dependencies = [
  "async-trait",
  "getrandom 0.2.11",
  "http 1.1.0",
- "matchit",
+ "matchit 0.8.4",
  "opentelemetry",
  "reqwest",
  "reqwest-middleware",
@@ -6052,6 +6222,19 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustix"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
+dependencies = [
+ "bitflags 2.8.0",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.9.4",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "rustls"
 version = "0.21.12"
@@ -6234,7 +6417,7 @@ dependencies = [
  "postgres_backend",
  "postgres_ffi",
  "postgres_versioninfo",
- "pprof",
+ "pprof 0.14.0",
  "pq_proto",
  "rand 0.8.5",
  "regex",
@@ -6822,6 +7005,18 @@ name = "spin"
 version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+dependencies = [
+ "lock_api",
+]
+
+[[package]]
+name = "spin"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591"
+dependencies = [
+ "lock_api",
+]
 
 [[package]]
 name = "spinning_top"
@@ -6922,12 +7117,12 @@ dependencies = [
  "hyper 0.14.30",
  "itertools 0.10.5",
  "json-structural-diff",
+ "lasso",
  "measured",
  "metrics",
  "once_cell",
  "pageserver_api",
  "pageserver_client",
- "paracord",
  "postgres_connection",
  "posthog_client_lite",
  "rand 0.8.5",
@@ -7016,7 +7211,6 @@ dependencies = [
  "pageserver_api",
  "pageserver_client",
  "reqwest",
- "safekeeper_api",
  "serde_json",
  "storage_controller_client",
  "tokio",
@@ -7183,14 +7377,14 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.14.0"
+version = "3.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
+checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
 dependencies = [
- "cfg-if",
  "fastrand 2.2.0",
+ "getrandom 0.3.3",
  "once_cell",
- "rustix",
+ "rustix 1.0.7",
  "windows-sys 0.59.0",
 ]
 
@@ -7684,16 +7878,25 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
 dependencies = [
+ "async-stream",
  "async-trait",
+ "axum 0.7.9",
  "base64 0.22.1",
  "bytes",
+ "h2 0.4.4",
  "http 1.1.0",
  "http-body 1.0.0",
  "http-body-util",
+ "hyper 1.4.1",
+ "hyper-timeout",
+ "hyper-util",
  "percent-encoding",
  "pin-project",
  "prost 0.13.5",
+ "socket2",
+ "tokio",
  "tokio-stream",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -7706,7 +7909,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
 dependencies = [
  "async-trait",
- "axum",
+ "axum 0.8.1",
  "base64 0.22.1",
  "bytes",
  "flate2",
@@ -7767,11 +7970,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap 1.9.3",
  "pin-project",
  "pin-project-lite",
+ "rand 0.8.5",
+ "slab",
  "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -8042,12 +8250,6 @@ dependencies = [
  "static_assertions",
 ]
 
-[[package]]
-name = "typed-arena"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
-
 [[package]]
 name = "typed-json"
 version = "0.1.1"
@@ -8209,7 +8411,7 @@ dependencies = [
  "pem",
  "pin-project-lite",
  "postgres_connection",
- "pprof",
+ "pprof 0.14.0",
  "pq_proto",
  "rand 0.8.5",
  "regex",
@@ -8261,7 +8463,7 @@ name = "vm_monitor"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "axum",
+ "axum 0.8.1",
  "cgroups-rs",
  "clap",
  "futures",
@@ -8316,7 +8518,7 @@ dependencies = [
  "pageserver_api",
  "postgres_ffi",
  "postgres_ffi_types",
- "pprof",
+ "pprof 0.14.0",
  "prost 0.13.5",
  "remote_storage",
  "serde",
@@ -8346,7 +8548,7 @@ name = "walproposer"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "bindgen",
+ "bindgen 0.71.1",
  "postgres_ffi",
  "utils",
 ]
@@ -8373,6 +8575,15 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasi"
+version = "0.14.2+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
 [[package]]
 name = "wasite"
 version = "0.1.0"
@@ -8502,6 +8713,18 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "which"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+dependencies = [
+ "either",
+ "home",
+ "once_cell",
+ "rustix 0.38.41",
+]
+
 [[package]]
 name = "whoami"
 version = "1.5.1"
@@ -8730,6 +8953,15 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
+dependencies = [
+ "bitflags 2.8.0",
+]
+
 [[package]]
 name = "workspace_hack"
 version = "0.1.0"
@@ -8737,17 +8969,19 @@ dependencies = [
  "ahash",
  "anstream",
  "anyhow",
- "axum",
- "axum-core",
+ "axum 0.8.1",
+ "axum-core 0.5.0",
  "base64 0.21.7",
  "base64ct",
  "bytes",
  "camino",
  "cc",
  "chrono",
+ "clang-sys",
  "clap",
  "clap_builder",
  "const-oid",
+ "criterion",
  "crypto-bigint 0.5.5",
  "der 0.7.8",
  "deranged",
@@ -8774,6 +9008,7 @@ dependencies = [
  "hyper 0.14.30",
  "hyper 1.4.1",
  "hyper-util",
+ "indexmap 1.9.3",
  "indexmap 2.9.0",
  "itertools 0.12.1",
  "lazy_static",
@@ -8790,9 +9025,7 @@ dependencies = [
  "num-iter",
  "num-rational",
  "num-traits",
- "once_cell",
  "p256 0.13.2",
- "paracord",
  "parquet",
  "prettyplease",
  "proc-macro2",
@@ -8813,6 +9046,7 @@ dependencies = [
  "sha2",
  "signature 2.2.0",
  "smallvec",
+ "spin 0.9.8",
  "spki 0.7.3",
  "stable_deref_trait",
  "subtle",
@@ -8827,6 +9061,7 @@ dependencies = [
  "tokio-stream",
  "tokio-util",
  "toml_edit",
+ "tonic 0.12.3",
  "tower 0.5.2",
  "tracing",
  "tracing-core",

Cargo.toml

@@ -128,11 +128,12 @@ itertools = "0.10"
 itoa = "1.0.11"
 jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
 jsonwebtoken = "9"
+lasso = "0.7"
 libc = "0.2"
+libproc = "0.14"
 md5 = "0.7.0"
-measured = { version = "0.0.23", features = ["paracord"] }
-measured-process = { version = "0.0.23" }
-measured-tokio = { version = "0.0.23" }
+measured = { version = "0.0.22", features=["lasso"] }
+measured-process = { version = "0.0.22" }
 memoffset = "0.9"
 nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
 # Do not update to >= 7.0.0, at least. The update will have a significant impact
@@ -145,7 +146,6 @@ opentelemetry = "0.27"
 opentelemetry_sdk = "0.27"
 opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] }
 opentelemetry-semantic-conventions = "0.27"
-paracord = { version = "0.1.0", features = ["serde"] }
 parking_lot = "0.12"
 parquet = { version = "53", default-features = false, features = ["zstd"] }
 parquet_derive = "53"
@@ -279,6 +279,7 @@ safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
 safekeeper_client = { path = "./safekeeper/client" }
 storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
 storage_controller_client = { path = "./storage_controller/client" }
+tempfile = "3"
 tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
 tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
 utils = { version = "0.1", path = "./libs/utils/" }

Dockerfile

@@ -109,6 +109,8 @@ RUN set -e \
         libreadline-dev \
         libseccomp-dev \
         ca-certificates \
+        bpfcc-tools \
+        sudo \
         openssl \
         unzip \
         curl \

build-tools.Dockerfile

@@ -61,6 +61,9 @@ RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
         libpq5 \
         libpq-dev \
         libzstd-dev \
+        linux-perf \
+        bpfcc-tools \
+        linux-headers-$(case "$(uname -m)" in x86_64) echo amd64;; aarch64) echo arm64;; esac) \
         postgresql-16 \
         postgresql-server-dev-16 \
         postgresql-common  \
@@ -105,15 +108,21 @@ RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
 #
 # 'gdb' is included so that we get backtraces of core dumps produced in
 # regression tests
-RUN set -e \
+RUN set -ex \
+    && KERNEL_VERSION="$(uname -r | cut -d'-' -f1 | sed 's/\.0$//')" \
+    && echo KERNEL_VERSION=${KERNEL_VERSION} >> /etc/environment \
+    && KERNEL_ARCH=$(uname -m | awk '{ if ($1 ~ /^(x86_64|i[3-6]86)$/) print "x86"; else if ($1 ~ /^(aarch64|arm.*)$/) print "aarch"; else print $1 }') \
+    && echo KERNEL_ARCH=${KERNEL_ARCH} >> /etc/environment \
     && apt update \
     && apt install -y \
         autoconf \
         automake \
+        bc \
         bison \
         build-essential \
         ca-certificates \
         cmake \
+        cpio \
         curl \
         flex \
         gdb \
@@ -122,8 +131,10 @@ RUN set -e \
         gzip \
         jq \
         jsonnet \
+        kmod \
         libcurl4-openssl-dev \
         libbz2-dev \
+        libelf-dev \
         libffi-dev \
         liblzma-dev \
         libncurses5-dev \
@@ -137,6 +148,11 @@ RUN set -e \
         libxml2-dev \
         libxmlsec1-dev \
         libxxhash-dev \
+        linux-perf \
+        bpfcc-tools \
+        libbpfcc \
+        libbpfcc-dev \
+        linux-headers-$(case "$(uname -m)" in x86_64) echo amd64;; aarch64) echo arm64;; esac) \
         lsof \
         make \
         netcat-openbsd \
@@ -144,6 +160,8 @@ RUN set -e \
         openssh-client \
         parallel \
         pkg-config \
+        rsync \
+        sudo \
         unzip \
         wget \
         xz-utils \
@@ -198,6 +216,8 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /
 # Configure sudo & docker
 RUN usermod -aG sudo nonroot && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    mkdir -p /etc/sudoers.d && \
+    echo 'nonroot ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/nonroot && \
     usermod -aG docker nonroot
 
 # AWS CLI

compute/compute-node.Dockerfile

@@ -149,6 +149,9 @@ RUN case $DEBIAN_VERSION in \
     ninja-build git autoconf automake libtool build-essential bison flex libreadline-dev \
     zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget ca-certificates pkg-config libssl-dev \
     libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd curl unzip g++ \
+    bpfcc-tools \
+    libbpfcc \
+    libbpfcc-dev \
     libclang-dev \
     jsonnet \
     $VERSION_INSTALLS \
@@ -1988,6 +1991,10 @@ RUN apt update && \
         locales \
         lsof \
         procps \
+        bpfcc-tools \
+        libbpfcc \
+        libbpfcc-dev \
+        libclang-dev \
         rsyslog-gnutls \
         screen \
         tcpdump \

compute/vm-image-spec-bookworm.yaml

@@ -39,6 +39,14 @@ commands:
     user: nobody
     sysvInitAction: respawn
     shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499'
+  - name: enable-kernel-modules
+    user: root
+    sysvInitAction: sysinit
+    shell: mkdir -p /lib/ && ln -s /neonvm/tools/lib/modules /lib/
+  - name: enable-bpfs
+    user: root
+    sysvInitAction: sysinit
+    shell: mkdir -p /sys/kernel/debug && mount -t debugfs debugfs /sys/kernel/debug && mount -t bpf bpf /sys/fs/bpf && chmod 755 /sys/fs/bpf
   # Rsyslog by default creates a unix socket under /dev/log . That's where Postgres sends logs also.
   # We run syslog with postgres user so it can't create /dev/log. Instead we configure rsyslog to
   # use a different path for the socket. The symlink actually points to our custom path.
@@ -65,7 +73,7 @@ files:
       # regardless of hostname (ALL)
       #
       # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd, /neonvm/tools/bin/perf, /usr/sbin/profile-bpfcc
   - filename: cgconfig.conf
     content: |
       # Configuration for cgroups in VM compute nodes
@@ -152,6 +160,8 @@ merge: |
   RUN set -e \
       && chmod 0644 /etc/cgconfig.conf
 
+  ENV PERF_BINARY_PATH=/neonvm/tools/bin/perf
+
 
   COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
   RUN chmod 0666 /etc/compute_rsyslog.conf

compute/vm-image-spec-bullseye.yaml

@@ -39,6 +39,14 @@ commands:
     user: nobody
     sysvInitAction: respawn
     shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499'
+  - name: enable-kernel-modules
+    user: root
+    sysvInitAction: sysinit
+    shell: mkdir -p /lib/ && ln -s /neonvm/tools/lib/modules /lib/
+  - name: enable-bpfs
+    user: root
+    sysvInitAction: sysinit
+    shell: mkdir -p /sys/kernel/debug && mount -t debugfs debugfs /sys/kernel/debug && mount -t bpf bpf /sys/fs/bpf && chmod 755 /sys/fs/bpf
   # Rsyslog by default creates a unix socket under /dev/log . That's where Postgres sends logs also.
   # We run syslog with postgres user so it can't create /dev/log. Instead we configure rsyslog to
   # use a different path for the socket. The symlink actually points to our custom path.
@@ -65,7 +73,7 @@ files:
       # regardless of hostname (ALL)
       #
       # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd, /neonvm/tools/bin/perf, /usr/sbin/profile-bpfcc
   - filename: cgconfig.conf
     content: |
       # Configuration for cgroups in VM compute nodes
@@ -148,6 +156,8 @@ merge: |
   RUN set -e \
       && chmod 0644 /etc/cgconfig.conf
 
+  ENV PERF_BINARY_PATH=/neonvm/tools/bin/perf
+
   COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
   RUN chmod 0666 /etc/compute_rsyslog.conf
   RUN mkdir /var/log/rsyslog && chown -R postgres /var/log/rsyslog

compute_tools/Cargo.toml

@@ -31,6 +31,7 @@ hostname-validator = "1.1"
 indexmap.workspace = true
 itertools.workspace = true
 jsonwebtoken.workspace = true
+libproc.workspace = true
 metrics.workspace = true
 nix.workspace = true
 notify.workspace = true
@@ -49,6 +50,7 @@ serde_with.workspace = true
 serde_json.workspace = true
 signal-hook.workspace = true
 tar.workspace = true
+tempfile.workspace = true
 tower.workspace = true
 tower-http.workspace = true
 tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
@@ -78,3 +80,10 @@ zstd = "0.13"
 bytes = "1.0"
 rust-ini = "0.20.0"
 rlimit = "0.10.1"
+
+inferno = { version = "0.12", default-features = false, features = [
+    "multithreaded",
+    "nameattr",
+] }
+pprof = { version = "0.15", features = ["protobuf-codec", "flamegraph"] }
+prost = "0.12"

compute_tools/src/compute.rs

@@ -371,7 +371,9 @@ fn maybe_cgexec(cmd: &str) -> Command {
     }
 }
 
-struct PostgresHandle {
+/// A handle to the Postgres process that is running in the compute
+/// node.
+pub struct PostgresHandle {
     postgres: std::process::Child,
     log_collector: JoinHandle<Result<()>>,
 }

compute_tools/src/http/routes/mod.rs

@@ -15,6 +15,7 @@ pub(in crate::http) mod lfc;
 pub(in crate::http) mod metrics;
 pub(in crate::http) mod metrics_json;
 pub(in crate::http) mod promote;
+pub(in crate::http) mod profile;
 pub(in crate::http) mod status;
 pub(in crate::http) mod terminate;
 

compute_tools/src/http/routes/profile.rs

@@ -0,0 +1,217 @@
+//! Contains the route for profiling the compute.
+//!
+//! Profiling the compute means generating a pprof profile of the
+//! postgres processes.
+//!
+//! The profiling is done using the `perf` tool, which is expected to be
+//! available somewhere in `$PATH`.
+use std::sync::atomic::Ordering;
+
+use axum::Json;
+use axum::response::IntoResponse;
+use http::StatusCode;
+use nix::unistd::Pid;
+use once_cell::sync::Lazy;
+use tokio::sync::Mutex;
+
+use crate::http::JsonResponse;
+
+static CANCEL_CHANNEL: Lazy<Mutex<Option<tokio::sync::broadcast::Sender<()>>>> =
+    Lazy::new(|| Mutex::new(None));
+
+fn default_sampling_frequency() -> u16 {
+    100
+}
+
+fn default_timeout_seconds() -> u8 {
+    5
+}
+
+fn deserialize_sampling_frequency<'de, D>(deserializer: D) -> Result<u16, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    use serde::Deserialize;
+
+    const MIN_SAMPLING_FREQUENCY: u16 = 1;
+    const MAX_SAMPLING_FREQUENCY: u16 = 1000;
+
+    let value = u16::deserialize(deserializer)?;
+
+    if !(MIN_SAMPLING_FREQUENCY..=MAX_SAMPLING_FREQUENCY).contains(&value) {
+        return Err(serde::de::Error::custom(format!(
+            "sampling_frequency must be between {MIN_SAMPLING_FREQUENCY} and {MAX_SAMPLING_FREQUENCY}, got {value}"
+        )));
+    }
+    Ok(value)
+}
+
+fn deserialize_profiling_timeout<'de, D>(deserializer: D) -> Result<u8, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    use serde::Deserialize;
+
+    const MIN_TIMEOUT_SECONDS: u8 = 1;
+    const MAX_TIMEOUT_SECONDS: u8 = 60;
+
+    let value = u8::deserialize(deserializer)?;
+
+    if !(MIN_TIMEOUT_SECONDS..=MAX_TIMEOUT_SECONDS).contains(&value) {
+        return Err(serde::de::Error::custom(format!(
+            "timeout_seconds must be between {MIN_TIMEOUT_SECONDS} and {MAX_TIMEOUT_SECONDS}, got {value}"
+        )));
+    }
+    Ok(value)
+}
+
+/// Request parameters for profiling the compute.
+#[derive(Debug, Clone, serde::Deserialize)]
+pub(in crate::http) struct ProfileRequest {
+    /// The profiling tool to use, currently only `perf` is supported.
+    profiler: crate::profiling::ProfileGenerator,
+    #[serde(default = "default_sampling_frequency")]
+    #[serde(deserialize_with = "deserialize_sampling_frequency")]
+    sampling_frequency: u16,
+    #[serde(default = "default_timeout_seconds")]
+    #[serde(deserialize_with = "deserialize_profiling_timeout")]
+    timeout_seconds: u8,
+    #[serde(default)]
+    archive: bool,
+}
+
+/// The HTTP request handler for reporting the profiling status of
+/// the compute.
+pub(in crate::http) async fn profile_status() -> impl IntoResponse {
+    tracing::info!("Profile status request received.");
+
+    let cancel_channel = CANCEL_CHANNEL.lock().await;
+
+    if let Some(tx) = cancel_channel.as_ref() {
+        if tx.receiver_count() > 0 {
+            return JsonResponse::create_response(
+                StatusCode::OK,
+                "Profiling is currently in progress.",
+            );
+        }
+    }
+
+    JsonResponse::create_response(StatusCode::NO_CONTENT, "Profiling is not in progress.")
+}
+
+/// The HTTP request handler for stopping profiling the compute.
+pub(in crate::http) async fn profile_stop() -> impl IntoResponse {
+    tracing::info!("Profile stop request received.");
+
+    match CANCEL_CHANNEL.lock().await.take() {
+        Some(tx) => {
+            if tx.send(()).is_err() {
+                tracing::error!("Failed to send cancellation signal.");
+                return JsonResponse::create_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    "Failed to send cancellation signal",
+                );
+            }
+            JsonResponse::create_response(StatusCode::OK, "Profiling stopped successfully.")
+        }
+        None => JsonResponse::create_response(
+            StatusCode::PRECONDITION_FAILED,
+            "Profiling is not in progress, there is nothing to stop.",
+        ),
+    }
+}
+
+/// The HTTP request handler for starting profiling the compute.
+pub(in crate::http) async fn profile_start(
+    Json(request): Json<ProfileRequest>,
+) -> impl IntoResponse {
+    tracing::info!("Profile start request received: {request:?}");
+
+    let tx = tokio::sync::broadcast::Sender::<()>::new(1);
+
+    {
+        let mut cancel_channel = CANCEL_CHANNEL.lock().await;
+
+        if cancel_channel.is_some() {
+            return JsonResponse::create_response(
+                StatusCode::CONFLICT,
+                "Profiling is already in progress.",
+            );
+        }
+        *cancel_channel = Some(tx.clone());
+    }
+
+    tracing::info!("Profiling will start with parameters: {request:?}");
+    let pg_pid = Pid::from_raw(crate::compute::PG_PID.load(Ordering::SeqCst) as _);
+
+    let run_with_sudo = !cfg!(feature = "testing");
+
+    let options = crate::profiling::ProfileGenerationOptions {
+        profiler: request.profiler,
+        run_with_sudo,
+        pids: [pg_pid].into_iter().collect(),
+        follow_forks: true,
+        sampling_frequency: request.sampling_frequency as u32,
+        blocklist_symbols: vec![
+            "libc".to_owned(),
+            "libgcc".to_owned(),
+            "pthread".to_owned(),
+            "vdso".to_owned(),
+        ],
+        archive: request.archive,
+    };
+
+    let options = crate::profiling::ProfileGenerationTaskOptions {
+        options,
+        timeout: std::time::Duration::from_secs(request.timeout_seconds as u64),
+        should_stop: Some(tx),
+    };
+
+    let pprof_data = crate::profiling::generate_pprof_profile(options).await;
+
+    if CANCEL_CHANNEL.lock().await.take().is_none() {
+        tracing::error!("Profiling was cancelled from another request.");
+
+        return JsonResponse::create_response(
+            StatusCode::NO_CONTENT,
+            "Profiling was cancelled from another request.",
+        );
+    }
+
+    let pprof_data = match pprof_data {
+        Ok(data) => data,
+        Err(e) => {
+            tracing::error!(error = ?e, "failed to generate pprof data");
+            return JsonResponse::create_response(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("Failed to generate pprof data: {e:?}"),
+            );
+        }
+    };
+
+    tracing::info!("Profiling has completed successfully.");
+
+    let mut headers = http::HeaderMap::new();
+
+    if request.archive {
+        headers.insert(
+            http::header::CONTENT_TYPE,
+            http::HeaderValue::from_static("application/gzip"),
+        );
+        headers.insert(
+            http::header::CONTENT_DISPOSITION,
+            http::HeaderValue::from_static("attachment; filename=\"profile.pb.gz\""),
+        );
+    } else {
+        headers.insert(
+            http::header::CONTENT_TYPE,
+            http::HeaderValue::from_static("application/octet-stream"),
+        );
+        headers.insert(
+            http::header::CONTENT_DISPOSITION,
+            http::HeaderValue::from_static("attachment; filename=\"profile.pb\""),
+        );
+    }
+
+    (headers, pprof_data.0).into_response()
+}

compute_tools/src/http/server.rs

@@ -27,6 +27,7 @@ use super::{
     },
 };
 use crate::compute::ComputeNode;
+use crate::http::routes::profile;
 
 /// `compute_ctl` has two servers: internal and external. The internal server
 /// binds to the loopback interface and handles communication from clients on
@@ -81,8 +82,14 @@ impl From<&Server> for Router<Arc<ComputeNode>> {
             Server::External {
                 config, compute_id, ..
             } => {
-                let unauthenticated_router =
-                    Router::<Arc<ComputeNode>>::new().route("/metrics", get(metrics::get_metrics));
+                let unauthenticated_router = Router::<Arc<ComputeNode>>::new()
+                    .route("/metrics", get(metrics::get_metrics))
+                    .route(
+                        "/profile/cpu",
+                        get(profile::profile_status)
+                            .post(profile::profile_start)
+                            .delete(profile::profile_stop),
+                    );
 
                 let authenticated_router = Router::<Arc<ComputeNode>>::new()
                     .route("/lfc/prewarm", get(lfc::prewarm_state).post(lfc::prewarm))

compute_tools/src/lib.rs

@@ -24,6 +24,7 @@ pub mod monitor;
 pub mod params;
 pub mod pg_helpers;
 pub mod pgbouncer;
+pub mod profiling;
 pub mod rsyslog;
 pub mod spec;
 mod spec_apply;

control_plane/src/broker.rs

@@ -36,7 +36,7 @@ impl StorageBroker {
     pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
         let broker = &self.env.broker;
 
-        println!("Starting neon broker at {}", broker.client_url());
+        print!("Starting neon broker at {}", broker.client_url());
 
         let mut args = Vec::new();
 

control_plane/src/pageserver.rs

@@ -303,7 +303,7 @@ impl PageServerNode {
     async fn start_node(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
         // TODO: using a thread here because start_process() is not async but we need to call check_status()
         let datadir = self.repo_path();
-        println!(
+        print!(
             "Starting pageserver node {} at '{}' in {:?}, retrying for {:?}",
             self.conf.id,
             self.pg_connection_config.raw_address(),

control_plane/src/safekeeper.rs

@@ -127,7 +127,7 @@ impl SafekeeperNode {
         extra_opts: &[String],
         retry_timeout: &Duration,
     ) -> anyhow::Result<()> {
-        println!(
+        print!(
             "Starting safekeeper at '{}' in '{}', retrying for {:?}",
             self.pg_connection_config.raw_address(),
             self.datadir_path().display(),

control_plane/src/storage_controller.rs

@@ -660,7 +660,7 @@ impl StorageController {
             ));
         }
 
-        println!("Starting storage controller at {scheme}://{host}:{listen_port}");
+        println!("Starting storage controller");
 
         background_process::start_process(
             COMMAND,

control_plane/storcon_cli/Cargo.toml

@@ -14,7 +14,6 @@ humantime.workspace = true
 pageserver_api.workspace = true
 pageserver_client.workspace = true
 reqwest.workspace = true
-safekeeper_api.workspace=true
 serde_json = { workspace = true, features = ["raw_value"] }
 storage_controller_client.workspace = true
 tokio.workspace = true

control_plane/storcon_cli/src/main.rs

@@ -11,7 +11,7 @@ use pageserver_api::controller_api::{
     PlacementPolicy, SafekeeperDescribeResponse, SafekeeperSchedulingPolicyRequest,
     ShardSchedulingPolicy, ShardsPreferredAzsRequest, ShardsPreferredAzsResponse,
     SkSchedulingPolicy, TenantCreateRequest, TenantDescribeResponse, TenantPolicyRequest,
-    TenantShardMigrateRequest, TenantShardMigrateResponse, TimelineSafekeeperMigrateRequest,
+    TenantShardMigrateRequest, TenantShardMigrateResponse,
 };
 use pageserver_api::models::{
     EvictionPolicy, EvictionPolicyLayerAccessThreshold, ShardParameters, TenantConfig,
@@ -21,7 +21,6 @@ use pageserver_api::models::{
 use pageserver_api::shard::{ShardStripeSize, TenantShardId};
 use pageserver_client::mgmt_api::{self};
 use reqwest::{Certificate, Method, StatusCode, Url};
-use safekeeper_api::models::TimelineLocateResponse;
 use storage_controller_client::control_api::Client;
 use utils::id::{NodeId, TenantId, TimelineId};
 
@@ -280,23 +279,6 @@ enum Command {
         #[arg(long)]
         concurrency: Option<usize>,
     },
-    /// Locate safekeepers for a timeline from the storcon DB.
-    TimelineLocate {
-        #[arg(long)]
-        tenant_id: TenantId,
-        #[arg(long)]
-        timeline_id: TimelineId,
-    },
-    /// Migrate a timeline to a new set of safekeepers
-    TimelineSafekeeperMigrate {
-        #[arg(long)]
-        tenant_id: TenantId,
-        #[arg(long)]
-        timeline_id: TimelineId,
-        /// Example: --new-sk-set 1,2,3
-        #[arg(long, required = true, value_delimiter = ',')]
-        new_sk_set: Vec<NodeId>,
-    },
 }
 
 #[derive(Parser)]
@@ -1342,7 +1324,7 @@ async fn main() -> anyhow::Result<()> {
             concurrency,
         } => {
             let mut path = format!(
-                "v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/download_heatmap_layers",
+                "/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/download_heatmap_layers",
             );
 
             if let Some(c) = concurrency {
@@ -1353,41 +1335,6 @@ async fn main() -> anyhow::Result<()> {
                 .dispatch::<(), ()>(Method::POST, path, None)
                 .await?;
         }
-        Command::TimelineLocate {
-            tenant_id,
-            timeline_id,
-        } => {
-            let path = format!("debug/v1/tenant/{tenant_id}/timeline/{timeline_id}/locate");
-
-            let resp = storcon_client
-                .dispatch::<(), TimelineLocateResponse>(Method::GET, path, None)
-                .await?;
-
-            let sk_set = resp.sk_set.iter().map(|id| id.0 as i64).collect::<Vec<_>>();
-            let new_sk_set = resp
-                .new_sk_set
-                .as_ref()
-                .map(|ids| ids.iter().map(|id| id.0 as i64).collect::<Vec<_>>());
-
-            println!("generation = {}", resp.generation);
-            println!("sk_set = {sk_set:?}");
-            println!("new_sk_set = {new_sk_set:?}");
-        }
-        Command::TimelineSafekeeperMigrate {
-            tenant_id,
-            timeline_id,
-            new_sk_set,
-        } => {
-            let path = format!("v1/tenant/{tenant_id}/timeline/{timeline_id}/safekeeper_migrate");
-
-            storcon_client
-                .dispatch::<_, ()>(
-                    Method::POST,
-                    path,
-                    Some(TimelineSafekeeperMigrateRequest { new_sk_set }),
-                )
-                .await?;
-        }
     }
 
     Ok(())

deny.toml

@@ -52,7 +52,6 @@ exceptions = [
     # Zlib license has some restrictions if we decide to change sth
     { allow = ["Zlib"], name = "const_format_proc_macros", version = "*" },
     { allow = ["Zlib"], name = "const_format", version = "*" },
-    { allow = ["Zlib"], name = "foldhash", version = "*" },
 ]
 
 [licenses.private]

docs/continuous-profiling.md

@@ -0,0 +1,58 @@
+# Continuous Crofiling (Compute)
+
+The continuous profiling of the compute node is performed by `perf` or `bcc-tools`, the latter is preferred.
+
+The executables profiled are all the postgres-related ones only, excluding the actual compute code (Rust). This can be done as well but
+was not the main goal.
+
+## Tools
+
+The aforementioned tools are available within the same Docker image as
+the compute node itself, but the corresponding dependencies linux the
+linux kernel headers and the linux kernel itself are not and can't be
+for obvious reasons. To solve that, as we run the compute nodes as a
+virtual machine (qemu), we need to deliver these dependencies to it.
+This is done by the `autoscaling` part, which builds and deploys the
+kernel headers, needed modules, and the `perf` binary into an ext4-fs
+disk image, which is later attached to the VM and is symlinked to be
+made available for the compute node.
+
+## Output
+
+The output of the profiling is always a binary file in the same format
+of `pprof`. It can, however, be archived by `gzip` additionally, if the
+corresponding argument is provided in the JSON request.
+
+## REST API
+
+### Test profiling
+
+One can test the profiling after connecting to the VM and running:
+
+```sh
+curl -X POST -H "Content-Type: application/json" http://localhost:3080/profile/cpu -d '{"profiler": {"BccProfile": null}, "sampling_frequency": 99, "timeout_seconds": 5, "archive": false}' -v --output profile.pb
+```
+
+This uses the `Bcc` profiler and does not archive the output. The
+profiling data will be saved into the `profile.pb` file locally.
+
+**Only one profiling session can be run at a time.**
+
+To check the profiling status (to see whether it is already running or
+not), one can perform the `GET` request:
+
+```sh
+curl http://localhost:3080/profile/cpu -v
+```
+
+The profiling can be stopped by performing the `DELETE` request:
+
+```sh
+curl -X DELETE http://localhost:3080/profile/cpu -v
+```
+
+## Supported profiling data
+
+For now, only the CPU profiling is done and ther is no heap profiling.
+Also, only the postgres-related executables are tracked, the compute
+(Rust) part itself **is not tracked**.

libs/neon-shmem/Cargo.toml

@@ -6,8 +6,8 @@ license.workspace = true
 
 [dependencies]
 thiserror.workspace = true
-nix.workspace=true
+nix.workspace = true
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
 
 [target.'cfg(target_os = "macos")'.dependencies]
-tempfile = "3.14.0"
+tempfile = "3.20.0"

libs/pageserver_api/src/controller_api.rs

@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use utils::id::{NodeId, TenantId, TimelineId};
 use utils::lsn::Lsn;
 
-use crate::models::{PageserverUtilization, ShardParameters, TenantConfig, TimelineInfo};
+use crate::models::{PageserverUtilization, ShardParameters, TenantConfig};
 use crate::shard::{ShardStripeSize, TenantShardId};
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -126,13 +126,6 @@ pub struct TenantDescribeResponse {
     pub config: TenantConfig,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct TenantTimelineDescribeResponse {
-    pub shards: Vec<TimelineInfo>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_consistent_lsn: Option<Lsn>,
-}
-
 #[derive(Serialize, Deserialize, Debug)]
 pub struct NodeShardResponse {
     pub node_id: NodeId,

libs/pageserver_api/src/models.rs

@@ -1622,9 +1622,6 @@ pub struct TimelineInfo {
 
     /// Whether the timeline is invisible in synthetic size calculations.
     pub is_invisible: Option<bool>,
-    // HADRON: the largest LSN below which all page updates have been included in the image layers.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_consistent_lsn: Option<Lsn>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

libs/safekeeper_api/src/models.rs

@@ -11,7 +11,7 @@ use utils::id::{NodeId, TenantId, TenantTimelineId, TimelineId};
 use utils::lsn::Lsn;
 use utils::pageserver_feedback::PageserverFeedback;
 
-use crate::membership::{Configuration, SafekeeperGeneration};
+use crate::membership::Configuration;
 use crate::{ServerInfo, Term};
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -311,12 +311,3 @@ pub struct PullTimelineResponse {
     pub safekeeper_host: Option<String>,
     // TODO: add more fields?
 }
-
-/// Response to a timeline locate request.
-/// Storcon-only API.
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct TimelineLocateResponse {
-    pub generation: SafekeeperGeneration,
-    pub sk_set: Vec<NodeId>,
-    pub new_sk_set: Option<Vec<NodeId>>,
-}

pageserver/src/http/routes.rs

@@ -397,7 +397,6 @@ async fn build_timeline_info(
     timeline: &Arc<Timeline>,
     include_non_incremental_logical_size: bool,
     force_await_initial_logical_size: bool,
-    include_image_consistent_lsn: bool,
     ctx: &RequestContext,
 ) -> anyhow::Result<TimelineInfo> {
     crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id();
@@ -422,10 +421,6 @@ async fn build_timeline_info(
                 .await?,
         );
     }
-    // HADRON
-    if include_image_consistent_lsn {
-        info.image_consistent_lsn = Some(timeline.compute_image_consistent_lsn().await?);
-    }
     Ok(info)
 }
 
@@ -515,8 +510,6 @@ async fn build_timeline_info_common(
         is_invisible: Some(is_invisible),
 
         walreceiver_status,
-        // HADRON
-        image_consistent_lsn: None,
     };
     Ok(info)
 }
@@ -719,8 +712,6 @@ async fn timeline_list_handler(
         parse_query_param(&request, "include-non-incremental-logical-size")?;
     let force_await_initial_logical_size: Option<bool> =
         parse_query_param(&request, "force-await-initial-logical-size")?;
-    let include_image_consistent_lsn: Option<bool> =
-        parse_query_param(&request, "include-image-consistent-lsn")?;
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
 
     let state = get_state(&request);
@@ -741,7 +732,6 @@ async fn timeline_list_handler(
                 &timeline,
                 include_non_incremental_logical_size.unwrap_or(false),
                 force_await_initial_logical_size.unwrap_or(false),
-                include_image_consistent_lsn.unwrap_or(false),
                 &ctx,
             )
             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
@@ -770,9 +760,6 @@ async fn timeline_and_offloaded_list_handler(
         parse_query_param(&request, "include-non-incremental-logical-size")?;
     let force_await_initial_logical_size: Option<bool> =
         parse_query_param(&request, "force-await-initial-logical-size")?;
-    let include_image_consistent_lsn: Option<bool> =
-        parse_query_param(&request, "include-image-consistent-lsn")?;
-
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
 
     let state = get_state(&request);
@@ -793,7 +780,6 @@ async fn timeline_and_offloaded_list_handler(
                 &timeline,
                 include_non_incremental_logical_size.unwrap_or(false),
                 force_await_initial_logical_size.unwrap_or(false),
-                include_image_consistent_lsn.unwrap_or(false),
                 &ctx,
             )
             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
@@ -978,9 +964,6 @@ async fn timeline_detail_handler(
         parse_query_param(&request, "include-non-incremental-logical-size")?;
     let force_await_initial_logical_size: Option<bool> =
         parse_query_param(&request, "force-await-initial-logical-size")?;
-    // HADRON
-    let include_image_consistent_lsn: Option<bool> =
-        parse_query_param(&request, "include-image-consistent-lsn")?;
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
 
     // Logical size calculation needs downloading.
@@ -1001,7 +984,6 @@ async fn timeline_detail_handler(
             &timeline,
             include_non_incremental_logical_size.unwrap_or(false),
             force_await_initial_logical_size.unwrap_or(false),
-            include_image_consistent_lsn.unwrap_or(false),
             ctx,
         )
         .await
@@ -3661,7 +3643,6 @@ async fn activate_post_import_handler(
         let timeline_info = build_timeline_info(
             &timeline, false, // include_non_incremental_logical_size,
             false, // force_await_initial_logical_size
-            false, // include_image_consistent_lsn
             &ctx,
         )
         .await

pageserver/src/tenant.rs

@@ -12816,40 +12816,6 @@ mod tests {
                 },
             ]
         );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_get_force_image_creation_lsn() -> anyhow::Result<()> {
-        let tenant_conf = pageserver_api::models::TenantConfig {
-            pitr_interval: Some(Duration::from_secs(7 * 3600)),
-            image_layer_force_creation_period: Some(Duration::from_secs(3600)),
-            ..Default::default()
-        };
-
-        let tenant_id = TenantId::generate();
-
-        let harness = TenantHarness::create_custom(
-            "test_get_force_image_creation_lsn",
-            tenant_conf,
-            tenant_id,
-            ShardIdentity::unsharded(),
-            Generation::new(1),
-        )
-        .await?;
-        let (tenant, ctx) = harness.load().await;
-        let timeline = tenant
-            .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
-            .await?;
-        timeline.gc_info.write().unwrap().cutoffs.time = Some(Lsn(100));
-        {
-            let writer = timeline.writer().await;
-            writer.finish_write(Lsn(5000));
-        }
-
-        let image_creation_lsn = timeline.get_force_image_creation_lsn().unwrap();
-        assert_eq!(image_creation_lsn, Lsn(4300));
         Ok(())
     }
 }

pageserver/src/tenant/layer_map.rs

@@ -46,11 +46,10 @@
 mod historic_layer_coverage;
 mod layer_coverage;
 
-use std::collections::{BTreeMap, HashMap, VecDeque};
+use std::collections::{HashMap, VecDeque};
 use std::iter::Peekable;
 use std::ops::Range;
 use std::sync::Arc;
-use std::time::Instant;
 
 use anyhow::Result;
 use historic_layer_coverage::BufferedHistoricLayerCoverage;
@@ -905,103 +904,6 @@ impl LayerMap {
         max_stacked_deltas
     }
 
-    /* BEGIN_HADRON */
-    /**
-     * Compute the image consistent LSN, the largest LSN below which all pages have been redone successfully.
-     * It works by first finding the latest image layers and store them into a map. Then for each delta layer,
-     * find all overlapping image layers in order to potentially increase the image LSN in case there are gaps
-     * (e.g., if an image is created at LSN 100 but the delta layer spans LSN [150, 200], then we can increase
-     * image LSN to 150 because there is no WAL record in between).
-     * Finally, the image consistent LSN is computed by taking the minimum of all image layers.
-     */
-    pub fn compute_image_consistent_lsn(&self, disk_consistent_lsn: Lsn) -> Lsn {
-        struct ImageLayerInfo {
-            // creation LSN of the image layer
-            image_lsn: Lsn,
-            // the current minimum LSN of newer delta layers with overlapping key ranges
-            min_delta_lsn: Lsn,
-        }
-        let started_at = Instant::now();
-
-        let min_l0_deltas_lsn = {
-            let l0_deltas = self.level0_deltas();
-            l0_deltas
-                .iter()
-                .map(|layer| layer.get_lsn_range().start)
-                .min()
-                .unwrap_or(disk_consistent_lsn)
-        };
-        let global_key_range = Key::MIN..Key::MAX;
-
-        // step 1: collect all most recent image layers into a map
-        // map: end key to image_layer_info
-        let mut image_map: BTreeMap<Key, ImageLayerInfo> = BTreeMap::new();
-        for (img_range, img) in self.image_coverage(&global_key_range, disk_consistent_lsn) {
-            let img_lsn = img.map(|layer| layer.get_lsn_range().end).unwrap_or(Lsn(0));
-            image_map.insert(
-                img_range.end,
-                ImageLayerInfo {
-                    image_lsn: img_lsn,
-                    min_delta_lsn: min_l0_deltas_lsn,
-                },
-            );
-        }
-
-        // step 2: go through all delta layers, and update the image layer info with overlapping
-        // key ranges
-        for layer in self.historic.iter() {
-            if !layer.is_delta {
-                continue;
-            }
-            let delta_key_range = layer.get_key_range();
-            let delta_lsn_range = layer.get_lsn_range();
-            for (img_end_key, img_info) in image_map.range_mut(delta_key_range.start..Key::MAX) {
-                debug_assert!(img_end_key >= &delta_key_range.start);
-                if delta_lsn_range.end > img_info.image_lsn {
-                    // the delta layer includes WAL records after the image
-                    // it's possibel that the delta layer's start LSN < image LSN, which will be simply ignored by step 3
-                    img_info.min_delta_lsn =
-                        std::cmp::min(img_info.min_delta_lsn, delta_lsn_range.start);
-                }
-                if img_end_key >= &delta_key_range.end {
-                    // we have fully processed all overlapping image layers
-                    break;
-                }
-            }
-        }
-
-        // step 3, go through all image layers and find the image consistent LSN
-        let mut img_consistent_lsn = min_l0_deltas_lsn.checked_sub(Lsn(1)).unwrap();
-        let mut prev_key = Key::MIN;
-        for (img_key, img_info) in image_map {
-            tracing::debug!(
-                "Image layer {:?}:{} has min delta lsn {}",
-                Range {
-                    start: prev_key,
-                    end: img_key,
-                },
-                img_info.image_lsn,
-                img_info.min_delta_lsn,
-            );
-            let image_lsn = std::cmp::max(
-                img_info.image_lsn,
-                img_info.min_delta_lsn.checked_sub(Lsn(1)).unwrap_or(Lsn(0)),
-            );
-            img_consistent_lsn = std::cmp::min(img_consistent_lsn, image_lsn);
-            prev_key = img_key;
-        }
-        tracing::info!(
-            "computed image_consistent_lsn {} for disk_consistent_lsn {} in {}ms. Processed {} layrs in total.",
-            img_consistent_lsn,
-            disk_consistent_lsn,
-            started_at.elapsed().as_millis(),
-            self.historic.len()
-        );
-        img_consistent_lsn
-    }
-
-    /* END_HADRON */
-
     /// Return all L0 delta layers
     pub fn level0_deltas(&self) -> &Vec<Arc<PersistentLayerDesc>> {
         &self.l0_delta_layers
@@ -1677,138 +1579,6 @@ mod tests {
             LayerVisibilityHint::Visible
         ));
     }
-
-    /* BEGIN_HADRON */
-    #[test]
-    fn test_compute_image_consistent_lsn() {
-        let mut layer_map = LayerMap::default();
-
-        let disk_consistent_lsn = Lsn(1000);
-        // case 1: empty layer map
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(
-            disk_consistent_lsn.checked_sub(Lsn(1)).unwrap(),
-            image_consistent_lsn
-        );
-
-        // case 2: only L0 delta layer
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(100),
-                Lsn(900)..Lsn(990),
-                true,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(100),
-                Lsn(850)..Lsn(899),
-                true,
-            ));
-        }
-
-        // should use min L0 delta LSN - 1 as image consistent LSN
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(849), image_consistent_lsn);
-
-        // case 3: 3 images, no L1 delta
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(40),
-                Lsn(100)..Lsn(100),
-                false,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(40)..Key::from_i128(70),
-                Lsn(200)..Lsn(200),
-                false,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(70)..Key::from_i128(100),
-                Lsn(150)..Lsn(150),
-                false,
-            ));
-        }
-        // should use min L0 delta LSN - 1 as image consistent LSN
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(849), image_consistent_lsn);
-
-        // case 4: 3 images with 1 L1 delta
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(50),
-                Lsn(300)..Lsn(350),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(299), image_consistent_lsn);
-
-        // case 5: 3 images with 1 more L1 delta with smaller LSN
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(50)..Key::from_i128(72),
-                Lsn(200)..Lsn(300),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(199), image_consistent_lsn);
-
-        // case 6: 3 images with more newer L1 deltas (no impact on final results)
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(30),
-                Lsn(400)..Lsn(500),
-                true,
-            ));
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(35)..Key::from_i128(100),
-                Lsn(450)..Lsn(600),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(199), image_consistent_lsn);
-
-        // case 7: 3 images with more older L1 deltas (no impact on final results)
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(40),
-                Lsn(0)..Lsn(50),
-                true,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(50)..Key::from_i128(100),
-                Lsn(10)..Lsn(60),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(199), image_consistent_lsn);
-
-        // case 8: 3 images with one more L1 delta with overlapping LSN range
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(50),
-                Lsn(50)..Lsn(250),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(100), image_consistent_lsn);
-    }
-
-    /* END_HADRON */
 }
 
 #[cfg(test)]

pageserver/src/tenant/timeline.rs

@@ -351,6 +351,13 @@ pub struct Timeline {
     last_image_layer_creation_check_at: AtomicLsn,
     last_image_layer_creation_check_instant: std::sync::Mutex<Option<Instant>>,
 
+    // HADRON
+    /// If a key range has writes with LSN > force_image_creation_lsn, then we should force image layer creation
+    /// on this key range.
+    force_image_creation_lsn: AtomicLsn,
+    /// The last time instant when force_image_creation_lsn is computed.
+    force_image_creation_lsn_computed_at: std::sync::Mutex<Option<Instant>>,
+
     /// Current logical size of the "datadir", at the last LSN.
     current_logical_size: LogicalSize,
 
@@ -2847,7 +2854,7 @@ impl Timeline {
     }
 
     // HADRON
-    fn get_image_layer_force_creation_period(&self) -> Option<Duration> {
+    fn get_image_creation_timeout(&self) -> Option<Duration> {
         let tenant_conf = self.tenant_conf.load();
         tenant_conf
             .tenant_conf
@@ -3127,6 +3134,9 @@ impl Timeline {
                 repartition_threshold: 0,
                 last_image_layer_creation_check_at: AtomicLsn::new(0),
                 last_image_layer_creation_check_instant: Mutex::new(None),
+                // HADRON
+                force_image_creation_lsn: AtomicLsn::new(0),
+                force_image_creation_lsn_computed_at: std::sync::Mutex::new(None),
                 last_received_wal: Mutex::new(None),
                 rel_size_latest_cache: RwLock::new(HashMap::new()),
                 rel_size_snapshot_cache: Mutex::new(LruCache::new(relsize_snapshot_cache_capacity)),
@@ -5371,16 +5381,13 @@ impl Timeline {
         }
 
         // HADRON
-        // for child timelines, we consider all pages up to ancestor_LSN are redone successfully by the parent timeline
-        min_image_lsn = min_image_lsn.max(self.get_ancestor_lsn());
         if min_image_lsn < force_image_creation_lsn.unwrap_or(Lsn(0)) && max_deltas > 0 {
             info!(
-                "forcing image creation for partitioned range {}-{}. Min image LSN: {}, force image creation LSN: {}, num deltas: {}",
+                "forcing image creation for partitioned range {}-{}. Min image LSN: {}, force image creation LSN: {}",
                 partition.ranges[0].start,
                 partition.ranges[0].end,
                 min_image_lsn,
-                force_image_creation_lsn.unwrap(),
-                max_deltas
+                force_image_creation_lsn.unwrap()
             );
             return true;
         }
@@ -7146,19 +7153,6 @@ impl Timeline {
             .unwrap()
             .clone()
     }
-
-    /* BEGIN_HADRON */
-    pub(crate) async fn compute_image_consistent_lsn(&self) -> anyhow::Result<Lsn> {
-        let guard = self
-            .layers
-            .read(LayerManagerLockHolder::ComputeImageConsistentLsn)
-            .await;
-        let layer_map = guard.layer_map()?;
-        let disk_consistent_lsn = self.get_disk_consistent_lsn();
-
-        Ok(layer_map.compute_image_consistent_lsn(disk_consistent_lsn))
-    }
-    /* END_HADRON */
 }
 
 impl Timeline {

pageserver/src/tenant/timeline/compaction.rs

@@ -8,7 +8,7 @@ use std::cmp::min;
 use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
 use std::ops::{Deref, Range};
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime};
 
 use super::layer_manager::LayerManagerLockHolder;
 use super::{
@@ -34,6 +34,7 @@ use pageserver_api::models::{CompactInfoResponse, CompactKeyRange};
 use pageserver_api::shard::{ShardCount, ShardIdentity, TenantShardId};
 use pageserver_compaction::helpers::{fully_contains, overlaps_with};
 use pageserver_compaction::interface::*;
+use postgres_ffi::to_pg_timestamp;
 use serde::Serialize;
 use tokio::sync::{OwnedSemaphorePermit, Semaphore};
 use tokio_util::sync::CancellationToken;
@@ -46,6 +47,7 @@ use wal_decoder::models::value::Value;
 
 use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder};
 use crate::page_cache;
+use crate::pgdatadir_mapping::LsnForTimestamp;
 use crate::statvfs::Statvfs;
 use crate::tenant::checks::check_valid_layermap;
 use crate::tenant::gc_block::GcBlock;
@@ -1269,7 +1271,10 @@ impl Timeline {
         // Define partitioning schema if needed
 
         // HADRON
-        let force_image_creation_lsn = self.get_force_image_creation_lsn();
+        let force_image_creation_lsn = self
+            .get_or_compute_force_image_creation_lsn(cancel, ctx)
+            .await
+            .map_err(CompactionError::Other)?;
 
         // 1. L0 Compact
         let l0_outcome = {
@@ -1479,37 +1484,59 @@ impl Timeline {
     }
 
     /* BEGIN_HADRON */
-    // Get the force image creation LSN based on gc_cutoff_lsn.
-    // Note that this is an estimation and the workload rate may suddenly change. When that happens,
-    // the force image creation may be too early or too late, but eventually it should be able to catch up.
-    pub(crate) fn get_force_image_creation_lsn(self: &Arc<Self>) -> Option<Lsn> {
-        let image_creation_period = self.get_image_layer_force_creation_period()?;
-        let current_lsn = self.get_last_record_lsn();
-        let pitr_lsn = self.gc_info.read().unwrap().cutoffs.time?;
-        let pitr_interval = self.get_pitr_interval();
-        if pitr_lsn == Lsn::INVALID || pitr_interval.is_zero() {
-            tracing::warn!(
-                "pitr LSN/interval not found, skipping force image creation LSN calculation"
-            );
-            return None;
+    // Get the force image creation LSN. Compute it if the last computed LSN is too old.
+    async fn get_or_compute_force_image_creation_lsn(
+        self: &Arc<Self>,
+        cancel: &CancellationToken,
+        ctx: &RequestContext,
+    ) -> anyhow::Result<Option<Lsn>> {
+        const FORCE_IMAGE_CREATION_LSN_COMPUTE_INTERVAL: Duration = Duration::from_secs(10 * 60); // 10 minutes
+        let image_layer_force_creation_period = self.get_image_creation_timeout();
+        if image_layer_force_creation_period.is_none() {
+            return Ok(None);
         }
 
-        let delta_lsn = current_lsn.checked_sub(pitr_lsn).unwrap().0
-            * image_creation_period.as_secs()
-            / pitr_interval.as_secs();
-        let force_image_creation_lsn = current_lsn.checked_sub(delta_lsn).unwrap_or(Lsn(0));
-
-        tracing::info!(
-            "Tenant shard {} computed force_image_creation_lsn: {}. Current lsn: {}, image_layer_force_creation_period: {:?}, GC cutoff: {}, PITR interval: {:?}",
-            self.tenant_shard_id,
-            force_image_creation_lsn,
-            current_lsn,
-            image_creation_period,
-            pitr_lsn,
-            pitr_interval
-        );
-
-        Some(force_image_creation_lsn)
+        let image_layer_force_creation_period = image_layer_force_creation_period.unwrap();
+        let force_image_creation_lsn_computed_at =
+            *self.force_image_creation_lsn_computed_at.lock().unwrap();
+        if force_image_creation_lsn_computed_at.is_none()
+            || force_image_creation_lsn_computed_at.unwrap().elapsed()
+                > FORCE_IMAGE_CREATION_LSN_COMPUTE_INTERVAL
+        {
+            let now: SystemTime = SystemTime::now();
+            let timestamp = now
+                .checked_sub(image_layer_force_creation_period)
+                .ok_or_else(|| {
+                    anyhow::anyhow!(
+                        "image creation timeout is too large: {image_layer_force_creation_period:?}"
+                    )
+                })?;
+            let timestamp = to_pg_timestamp(timestamp);
+            let force_image_creation_lsn = match self
+                .find_lsn_for_timestamp(timestamp, cancel, ctx)
+                .await?
+            {
+                LsnForTimestamp::Present(lsn) | LsnForTimestamp::Future(lsn) => lsn,
+                _ => {
+                    let gc_lsn = *self.get_applied_gc_cutoff_lsn();
+                    tracing::info!(
+                        "no LSN found for timestamp {timestamp:?}, using latest GC cutoff LSN {}",
+                        gc_lsn
+                    );
+                    gc_lsn
+                }
+            };
+            self.force_image_creation_lsn
+                .store(force_image_creation_lsn);
+            *self.force_image_creation_lsn_computed_at.lock().unwrap() = Some(Instant::now());
+            tracing::info!(
+                "computed force image creation LSN: {}",
+                force_image_creation_lsn
+            );
+            Ok(Some(force_image_creation_lsn))
+        } else {
+            Ok(Some(self.force_image_creation_lsn.load()))
+        }
     }
     /* END_HADRON */
 

pageserver/src/tenant/timeline/layer_manager.rs

@@ -47,7 +47,6 @@ pub(crate) enum LayerManagerLockHolder {
     ImportPgData,
     DetachAncestor,
     Eviction,
-    ComputeImageConsistentLsn,
     #[cfg(test)]
     Testing,
 }

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand.
 
 [[package]]
 name = "aiohappyeyeballs"
@@ -2326,6 +2326,25 @@ files = [
     {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"},
 ]
 
+[[package]]
+name = "protobuf"
+version = "6.31.1"
+description = ""
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+    {file = "protobuf-6.31.1-cp310-abi3-win32.whl", hash = "sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9"},
+    {file = "protobuf-6.31.1-cp310-abi3-win_amd64.whl", hash = "sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447"},
+    {file = "protobuf-6.31.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402"},
+    {file = "protobuf-6.31.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39"},
+    {file = "protobuf-6.31.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6"},
+    {file = "protobuf-6.31.1-cp39-cp39-win32.whl", hash = "sha256:0414e3aa5a5f3ff423828e1e6a6e907d6c65c1d5b7e6e975793d5590bdeecc16"},
+    {file = "protobuf-6.31.1-cp39-cp39-win_amd64.whl", hash = "sha256:8764cf4587791e7564051b35524b72844f845ad0bb011704c3736cce762d8fe9"},
+    {file = "protobuf-6.31.1-py3-none-any.whl", hash = "sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e"},
+    {file = "protobuf-6.31.1.tar.gz", hash = "sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a"},
+]
+
 [[package]]
 name = "psutil"
 version = "5.9.4"
@@ -3309,6 +3328,18 @@ files = [
 [package.dependencies]
 cryptography = "*"
 
+[[package]]
+name = "types-protobuf"
+version = "6.30.2.20250516"
+description = "Typing stubs for protobuf"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+    {file = "types_protobuf-6.30.2.20250516-py3-none-any.whl", hash = "sha256:8c226d05b5e8b2623111765fa32d6e648bbc24832b4c2fddf0fa340ba5d5b722"},
+    {file = "types_protobuf-6.30.2.20250516.tar.gz", hash = "sha256:aecd1881770a9bb225ede66872ef7f0da4505edd0b193108edd9892e48d49a41"},
+]
+
 [[package]]
 name = "types-psutil"
 version = "5.9.5.12"
@@ -3847,4 +3878,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.1"
 python-versions = "^3.11"
-content-hash = "bd93313f110110aa53b24a3ed47ba2d7f60e2c658a79cdff7320fed1bb1b57b5"
+content-hash = "7cc735f57c2760db6c994575a98d4f0e2670497ad9e909135a3bc67d479f5edf"

proxy/Cargo.toml

@@ -48,13 +48,12 @@ indexmap = { workspace = true, features = ["serde"] }
 ipnet.workspace = true
 itertools.workspace = true
 itoa.workspace = true
-measured = { workspace = true, features = ["paracord"] }
-measured-tokio.workspace = true
+lasso = { workspace = true, features = ["multi-threaded"] }
+measured = { workspace = true, features = ["lasso"] }
 metrics.workspace = true
 once_cell.workspace = true
 opentelemetry = { workspace = true, features = ["trace"] }
 papaya = "0.2.0"
-paracord.workspace = true
 parking_lot.workspace = true
 parquet.workspace = true
 parquet_derive.workspace = true
@@ -128,4 +127,4 @@ rstest.workspace = true
 walkdir.workspace = true
 rand_distr = "0.4"
 tokio-postgres.workspace = true
-tracing-test = "0.2"
+tracing-test = "0.2"

proxy/src/auth/backend/jwt.rs

@@ -87,12 +87,7 @@ impl JwkCacheEntry {
         self.key_sets
             .values()
             // make sure our requested role has access to the key set
-            .filter(|key_set| {
-                key_set
-                    .role_names
-                    .iter()
-                    .any(|role| *role.as_str() == **role_name)
-            })
+            .filter(|key_set| key_set.role_names.iter().any(|role| **role == **role_name))
             // try and find the requested key-id in the key set
             .find_map(|key_set| {
                 key_set

proxy/src/auth/backend/local.rs

@@ -12,7 +12,7 @@ use crate::context::RequestContext;
 use crate::control_plane::NodeInfo;
 use crate::control_plane::messages::{ColdStartInfo, EndpointJwksResponse, MetricsAuxInfo};
 use crate::http;
-use crate::intern::{BranchIdInt, EndpointIdInt, ProjectIdInt};
+use crate::intern::{BranchIdTag, EndpointIdTag, InternId, ProjectIdTag};
 use crate::types::EndpointId;
 use crate::url::ApiUrl;
 
@@ -38,9 +38,9 @@ impl LocalBackend {
                 },
                 // TODO(conrad): make this better reflect compute info rather than endpoint info.
                 aux: MetricsAuxInfo {
-                    endpoint_id: EndpointIdInt::new("local"),
-                    project_id: ProjectIdInt::new("local"),
-                    branch_id: BranchIdInt::new("local"),
+                    endpoint_id: EndpointIdTag::get_interner().get_or_intern("local"),
+                    project_id: ProjectIdTag::get_interner().get_or_intern("local"),
+                    branch_id: BranchIdTag::get_interner().get_or_intern("local"),
                     compute_id: "local".into(),
                     cold_start_info: ColdStartInfo::WarmCached,
                 },

proxy/src/binary/local_proxy.rs

@@ -192,7 +192,6 @@ pub async fn run() -> anyhow::Result<()> {
             jemalloc,
             neon_metrics,
             proxy: crate::metrics::Metrics::get(),
-            tokio: measured_tokio::RuntimeCollector::current(),
         },
     ));
 

proxy/src/binary/proxy.rs

@@ -513,7 +513,6 @@ pub async fn run() -> anyhow::Result<()> {
             jemalloc,
             neon_metrics,
             proxy: crate::metrics::Metrics::get(),
-            tokio: measured_tokio::RuntimeCollector::current(),
         },
     ));
     maintenance_tasks.spawn(control_plane::mgmt::task_main(mgmt_listener));

proxy/src/cache/project_info.rs

@@ -194,7 +194,7 @@ impl ProjectInfoCacheImpl {
         &self,
         endpoint_id: &EndpointId,
     ) -> Option<Ref<'_, EndpointIdInt, EndpointInfo>> {
-        let endpoint_id = EndpointIdInt::try_new_existing(endpoint_id)?;
+        let endpoint_id = EndpointIdInt::get(endpoint_id)?;
         self.cache.get(&endpoint_id)
     }
 
@@ -204,7 +204,7 @@ impl ProjectInfoCacheImpl {
         role_name: &RoleName,
     ) -> Option<RoleAccessControl> {
         let valid_since = self.get_cache_times();
-        let role_name = RoleNameInt::try_new_existing(role_name)?;
+        let role_name = RoleNameInt::get(role_name)?;
         let endpoint_info = self.get_endpoint_cache(endpoint_id)?;
         endpoint_info.get_role_secret(role_name, valid_since)
     }
@@ -297,10 +297,10 @@ impl ProjectInfoCacheImpl {
     }
 
     pub fn maybe_invalidate_role_secret(&self, endpoint_id: &EndpointId, role_name: &RoleName) {
-        let Some(endpoint_id) = EndpointIdInt::try_new_existing(endpoint_id) else {
+        let Some(endpoint_id) = EndpointIdInt::get(endpoint_id) else {
             return;
         };
-        let Some(role_name) = RoleNameInt::try_new_existing(role_name) else {
+        let Some(role_name) = RoleNameInt::get(role_name) else {
             return;
         };
 

proxy/src/http/health_server.rs

@@ -71,8 +71,6 @@ pub struct AppMetrics {
     pub neon_metrics: NeonMetrics,
     #[metric(flatten)]
     pub proxy: &'static crate::metrics::Metrics,
-    #[metric(namespace = "tokio")]
-    pub tokio: measured_tokio::RuntimeCollector,
 }
 
 async fn prometheus_metrics_handler(

proxy/src/intern.rs

@@ -1,59 +1,272 @@
-use paracord::custom_key;
+use std::hash::BuildHasherDefault;
+use std::marker::PhantomData;
+use std::num::NonZeroUsize;
+use std::ops::Index;
+use std::sync::OnceLock;
+
+use lasso::{Capacity, MemoryLimits, Spur, ThreadedRodeo};
+use rustc_hash::FxHasher;
 
 use crate::types::{AccountId, BranchId, EndpointId, ProjectId, RoleName};
 
-custom_key!(pub struct RoleNameInt);
-custom_key!(pub struct EndpointIdInt);
-custom_key!(pub struct BranchIdInt);
-custom_key!(pub struct ProjectIdInt);
-custom_key!(pub struct AccountIdInt);
+pub trait InternId: Sized + 'static {
+    fn get_interner() -> &'static StringInterner<Self>;
+}
 
-impl From<&RoleName> for RoleNameInt {
-    fn from(value: &RoleName) -> Self {
-        RoleNameInt::new(value)
+pub struct StringInterner<Id> {
+    inner: ThreadedRodeo<Spur, BuildHasherDefault<FxHasher>>,
+    _id: PhantomData<Id>,
+}
+
+#[derive(PartialEq, Debug, Clone, Copy, Eq, Hash)]
+pub struct InternedString<Id> {
+    inner: Spur,
+    _id: PhantomData<Id>,
+}
+
+impl<Id: InternId> std::fmt::Display for InternedString<Id> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.as_str().fmt(f)
     }
 }
 
+impl<Id: InternId> InternedString<Id> {
+    pub(crate) fn as_str(&self) -> &'static str {
+        Id::get_interner().inner.resolve(&self.inner)
+    }
+    pub(crate) fn get(s: &str) -> Option<Self> {
+        Id::get_interner().get(s)
+    }
+}
+
+impl<Id: InternId> AsRef<str> for InternedString<Id> {
+    fn as_ref(&self) -> &str {
+        self.as_str()
+    }
+}
+
+impl<Id: InternId> std::ops::Deref for InternedString<Id> {
+    type Target = str;
+    fn deref(&self) -> &str {
+        self.as_str()
+    }
+}
+
+impl<'de, Id: InternId> serde::de::Deserialize<'de> for InternedString<Id> {
+    fn deserialize<D: serde::de::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
+        struct Visitor<Id>(PhantomData<Id>);
+        impl<Id: InternId> serde::de::Visitor<'_> for Visitor<Id> {
+            type Value = InternedString<Id>;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                formatter.write_str("a string")
+            }
+
+            fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                Ok(Id::get_interner().get_or_intern(v))
+            }
+        }
+        d.deserialize_str(Visitor::<Id>(PhantomData))
+    }
+}
+
+impl<Id: InternId> serde::Serialize for InternedString<Id> {
+    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
+        self.as_str().serialize(s)
+    }
+}
+
+impl<Id: InternId> StringInterner<Id> {
+    pub(crate) fn new() -> Self {
+        StringInterner {
+            inner: ThreadedRodeo::with_capacity_memory_limits_and_hasher(
+                Capacity::new(2500, NonZeroUsize::new(1 << 16).expect("value is nonzero")),
+                // unbounded
+                MemoryLimits::for_memory_usage(usize::MAX),
+                BuildHasherDefault::<FxHasher>::default(),
+            ),
+            _id: PhantomData,
+        }
+    }
+
+    #[cfg(test)]
+    fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    #[cfg(test)]
+    fn current_memory_usage(&self) -> usize {
+        self.inner.current_memory_usage()
+    }
+
+    pub(crate) fn get_or_intern(&self, s: &str) -> InternedString<Id> {
+        InternedString {
+            inner: self.inner.get_or_intern(s),
+            _id: PhantomData,
+        }
+    }
+
+    pub(crate) fn get(&self, s: &str) -> Option<InternedString<Id>> {
+        Some(InternedString {
+            inner: self.inner.get(s)?,
+            _id: PhantomData,
+        })
+    }
+}
+
+impl<Id: InternId> Index<InternedString<Id>> for StringInterner<Id> {
+    type Output = str;
+
+    fn index(&self, index: InternedString<Id>) -> &Self::Output {
+        self.inner.resolve(&index.inner)
+    }
+}
+
+impl<Id: InternId> Default for StringInterner<Id> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RoleNameTag;
+impl InternId for RoleNameTag {
+    fn get_interner() -> &'static StringInterner<Self> {
+        static ROLE_NAMES: OnceLock<StringInterner<RoleNameTag>> = OnceLock::new();
+        ROLE_NAMES.get_or_init(Default::default)
+    }
+}
+pub type RoleNameInt = InternedString<RoleNameTag>;
+impl From<&RoleName> for RoleNameInt {
+    fn from(value: &RoleName) -> Self {
+        RoleNameTag::get_interner().get_or_intern(value)
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct EndpointIdTag;
+impl InternId for EndpointIdTag {
+    fn get_interner() -> &'static StringInterner<Self> {
+        static ROLE_NAMES: OnceLock<StringInterner<EndpointIdTag>> = OnceLock::new();
+        ROLE_NAMES.get_or_init(Default::default)
+    }
+}
+pub type EndpointIdInt = InternedString<EndpointIdTag>;
 impl From<&EndpointId> for EndpointIdInt {
     fn from(value: &EndpointId) -> Self {
-        EndpointIdInt::new(value)
+        EndpointIdTag::get_interner().get_or_intern(value)
     }
 }
 impl From<EndpointId> for EndpointIdInt {
     fn from(value: EndpointId) -> Self {
-        EndpointIdInt::new(&value)
+        EndpointIdTag::get_interner().get_or_intern(&value)
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct BranchIdTag;
+impl InternId for BranchIdTag {
+    fn get_interner() -> &'static StringInterner<Self> {
+        static ROLE_NAMES: OnceLock<StringInterner<BranchIdTag>> = OnceLock::new();
+        ROLE_NAMES.get_or_init(Default::default)
+    }
+}
+pub type BranchIdInt = InternedString<BranchIdTag>;
 impl From<&BranchId> for BranchIdInt {
     fn from(value: &BranchId) -> Self {
-        BranchIdInt::new(value)
+        BranchIdTag::get_interner().get_or_intern(value)
     }
 }
 impl From<BranchId> for BranchIdInt {
     fn from(value: BranchId) -> Self {
-        BranchIdInt::new(&value)
+        BranchIdTag::get_interner().get_or_intern(&value)
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ProjectIdTag;
+impl InternId for ProjectIdTag {
+    fn get_interner() -> &'static StringInterner<Self> {
+        static ROLE_NAMES: OnceLock<StringInterner<ProjectIdTag>> = OnceLock::new();
+        ROLE_NAMES.get_or_init(Default::default)
+    }
+}
+pub type ProjectIdInt = InternedString<ProjectIdTag>;
 impl From<&ProjectId> for ProjectIdInt {
     fn from(value: &ProjectId) -> Self {
-        ProjectIdInt::new(value)
+        ProjectIdTag::get_interner().get_or_intern(value)
     }
 }
 impl From<ProjectId> for ProjectIdInt {
     fn from(value: ProjectId) -> Self {
-        ProjectIdInt::new(&value)
+        ProjectIdTag::get_interner().get_or_intern(&value)
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct AccountIdTag;
+impl InternId for AccountIdTag {
+    fn get_interner() -> &'static StringInterner<Self> {
+        static ROLE_NAMES: OnceLock<StringInterner<AccountIdTag>> = OnceLock::new();
+        ROLE_NAMES.get_or_init(Default::default)
+    }
+}
+pub type AccountIdInt = InternedString<AccountIdTag>;
 impl From<&AccountId> for AccountIdInt {
     fn from(value: &AccountId) -> Self {
-        AccountIdInt::new(value)
+        AccountIdTag::get_interner().get_or_intern(value)
     }
 }
 impl From<AccountId> for AccountIdInt {
     fn from(value: AccountId) -> Self {
-        AccountIdInt::new(&value)
+        AccountIdTag::get_interner().get_or_intern(&value)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::OnceLock;
+
+    use super::InternId;
+    use crate::intern::StringInterner;
+
+    struct MyId;
+    impl InternId for MyId {
+        fn get_interner() -> &'static StringInterner<Self> {
+            pub(crate) static ROLE_NAMES: OnceLock<StringInterner<MyId>> = OnceLock::new();
+            ROLE_NAMES.get_or_init(Default::default)
+        }
+    }
+
+    #[test]
+    fn push_many_strings() {
+        use rand::rngs::StdRng;
+        use rand::{Rng, SeedableRng};
+        use rand_distr::Zipf;
+
+        let endpoint_dist = Zipf::new(500000, 0.8).unwrap();
+        let endpoints = StdRng::seed_from_u64(272488357).sample_iter(endpoint_dist);
+
+        let interner = MyId::get_interner();
+
+        const N: usize = 100_000;
+        let mut verify = Vec::with_capacity(N);
+        for endpoint in endpoints.take(N) {
+            let endpoint = format!("ep-string-interning-{endpoint}");
+            let key = interner.get_or_intern(&endpoint);
+            verify.push((endpoint, key));
+        }
+
+        for (s, key) in verify {
+            assert_eq!(interner[key], s);
+        }
+
+        // 2031616/59861 = 34 bytes per string
+        assert_eq!(interner.len(), 59_861);
+        // will have other overhead for the internal hashmaps that are not accounted for.
+        assert_eq!(interner.current_memory_usage(), 2_031_616);
     }
 }

proxy/src/metrics.rs

@@ -1,5 +1,6 @@
 use std::sync::{Arc, OnceLock};
 
+use lasso::ThreadedRodeo;
 use measured::label::{
     FixedCardinalitySet, LabelGroupSet, LabelName, LabelSet, LabelValue, StaticLabelSet,
 };
@@ -10,7 +11,6 @@ use measured::{
     MetricGroup,
 };
 use metrics::{CounterPairAssoc, CounterPairVec, HyperLogLogVec};
-use paracord::ParaCord;
 use tokio::time::{self, Instant};
 
 use crate::control_plane::messages::ColdStartInfo;
@@ -222,7 +222,7 @@ pub enum CacheOutcome {
 #[derive(LabelGroup)]
 #[label(set = ConsoleRequestSet)]
 pub struct ConsoleRequest<'a> {
-    #[label(dynamic_with = ParaCord, default)]
+    #[label(dynamic_with = ThreadedRodeo, default)]
     pub request: &'a str,
 }
 
@@ -345,7 +345,7 @@ pub struct ConnectionFailuresBreakdownGroup {
 #[derive(LabelGroup, Copy, Clone)]
 #[label(set = RedisErrorsSet)]
 pub struct RedisErrors<'a> {
-    #[label(dynamic_with = ParaCord, default)]
+    #[label(dynamic_with = ThreadedRodeo, default)]
     pub channel: &'a str,
 }
 

proxy/src/types.rs

@@ -1,4 +1,4 @@
-use crate::intern::EndpointIdInt;
+use crate::intern::{EndpointIdInt, EndpointIdTag, InternId};
 
 macro_rules! smol_str_wrapper {
     ($name:ident) => {
@@ -85,7 +85,7 @@ impl EndpointId {
 
     #[must_use]
     pub fn normalize_intern(&self) -> EndpointIdInt {
-        EndpointIdInt::new(self.normalize_str())
+        EndpointIdTag::get_interner().get_or_intern(self.normalize_str())
     }
 }
 

pyproject.toml

@@ -10,6 +10,8 @@ psycopg2-binary = "^2.9.10"
 typing-extensions = "^4.12.2"
 PyJWT = {version = "^2.1.0", extras = ["crypto"]}
 requests = "^2.32.4"
+protobuf = "^6.31.1"
+types-protobuf="^6.30.2"
 pytest-xdist = "^3.3.1"
 asyncpg = "^0.30.0"
 aiopg = "^1.4.0"
@@ -63,6 +65,7 @@ build-backend = "poetry.core.masonry.api"
 exclude = [
     "^vendor/",
     "^target/",
+    "test_runner/regress/data/profile_pb2.py",
     "test_runner/performance/pgvector/loaddata.py",
 ]
 check_untyped_defs = true
@@ -94,6 +97,7 @@ target-version = "py311"
 extend-exclude = [
     "vendor/",
     "target/",
+    "test_runner/regress/data/profile_pb2.py",
     "test_runner/stubs/", # Autogenerated by mypy's stubgen
 ]
 line-length = 100 # this setting is rather guidance, it won't fail if it can't make the shorter

storage_controller/Cargo.toml

@@ -31,10 +31,10 @@ humantime.workspace = true
 humantime-serde.workspace = true
 itertools.workspace = true
 json-structural-diff.workspace = true
+lasso.workspace = true
 once_cell.workspace = true
 pageserver_api.workspace = true
 pageserver_client.workspace = true
-paracord.workspace = true
 postgres_connection.workspace = true
 posthog_client_lite.workspace = true
 rand.workspace = true
@@ -72,4 +72,4 @@ http-utils = { path = "../libs/http-utils/" }
 utils = { path = "../libs/utils/" }
 metrics = { path = "../libs/metrics/" }
 control_plane = { path = "../control_plane" }
-workspace_hack = { version = "0.1", path = "../workspace_hack" }
+workspace_hack = { version = "0.1", path = "../workspace_hack" }

storage_controller/src/http.rs

@@ -850,31 +850,6 @@ async fn handle_tenant_describe(
     json_response(StatusCode::OK, service.tenant_describe(tenant_id)?)
 }
 
-/* BEGIN_HADRON */
-async fn handle_tenant_timeline_describe(
-    service: Arc<Service>,
-    req: Request<Body>,
-) -> Result<Response<Body>, ApiError> {
-    check_permissions(&req, Scope::Scrubber)?;
-
-    let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?;
-    let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?;
-    match maybe_forward(req).await {
-        ForwardOutcome::Forwarded(res) => {
-            return res;
-        }
-        ForwardOutcome::NotForwarded(_req) => {}
-    };
-
-    json_response(
-        StatusCode::OK,
-        service
-            .tenant_timeline_describe(tenant_id, timeline_id)
-            .await?,
-    )
-}
-/* END_HADRON */
-
 async fn handle_tenant_list(
     service: Arc<Service>,
     req: Request<Body>,
@@ -2505,13 +2480,6 @@ pub fn make_router(
             )
         })
         // Timeline operations
-        .get("/control/v1/tenant/:tenant_id/timeline/:timeline_id", |r| {
-            tenant_service_handler(
-                r,
-                handle_tenant_timeline_describe,
-                RequestName("v1_tenant_timeline_describe"),
-            )
-        })
         .delete("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| {
             tenant_service_handler(
                 r,

storage_controller/src/metrics.rs

@@ -183,9 +183,9 @@ impl Default for StorageControllerMetrics {
 #[derive(measured::LabelGroup, Clone)]
 #[label(set = NodeLabelGroupSet)]
 pub(crate) struct NodeLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) az: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) node_id: &'a str,
 }
 
@@ -198,7 +198,7 @@ pub(crate) struct ReconcileCompleteLabelGroup {
 #[derive(measured::LabelGroup)]
 #[label(set = HttpRequestStatusLabelGroupSet)]
 pub(crate) struct HttpRequestStatusLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) path: &'a str,
     pub(crate) method: Method,
     pub(crate) status: StatusCode,
@@ -207,7 +207,7 @@ pub(crate) struct HttpRequestStatusLabelGroup<'a> {
 #[derive(measured::LabelGroup)]
 #[label(set = HttpRequestLatencyLabelGroupSet)]
 pub(crate) struct HttpRequestLatencyLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) path: &'a str,
     pub(crate) method: Method,
 }
@@ -215,9 +215,9 @@ pub(crate) struct HttpRequestLatencyLabelGroup<'a> {
 #[derive(measured::LabelGroup, Clone)]
 #[label(set = PageserverRequestLabelGroupSet)]
 pub(crate) struct PageserverRequestLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) pageserver_id: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) path: &'a str,
     pub(crate) method: Method,
 }
@@ -225,9 +225,9 @@ pub(crate) struct PageserverRequestLabelGroup<'a> {
 #[derive(measured::LabelGroup, Clone)]
 #[label(set = SafekeeperRequestLabelGroupSet)]
 pub(crate) struct SafekeeperRequestLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) safekeeper_id: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) path: &'a str,
     pub(crate) method: Method,
 }
@@ -254,11 +254,11 @@ pub(crate) struct LeadershipStatusGroup {
 #[derive(measured::LabelGroup, Clone)]
 #[label(set = ReconcileLongRunningLabelGroupSet)]
 pub(crate) struct ReconcileLongRunningLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) tenant_id: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) shard_number: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) sequence: &'a str,
 }
 
@@ -282,11 +282,11 @@ pub(crate) enum Method {
 #[derive(measured::LabelGroup, Clone)]
 #[label(set = SafekeeperReconcilerLabelGroupSet)]
 pub(crate) struct SafekeeperReconcilerLabelGroup<'a> {
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) sk_az: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) sk_node_id: &'a str,
-    #[label(dynamic_with = paracord::ParaCord, default)]
+    #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     pub(crate) sk_hostname: &'a str,
 }
 

storage_controller/src/pageserver_client.rs

@@ -86,23 +86,6 @@ impl PageserverClient {
         )
     }
 
-    /* BEGIN_HADRON */
-    pub(crate) async fn tenant_timeline_describe(
-        &self,
-        tenant_shard_id: &TenantShardId,
-        timeline_id: &TimelineId,
-    ) -> Result<TimelineInfo> {
-        measured_request!(
-            "tenant_timeline_describe",
-            crate::metrics::Method::Get,
-            &self.node_id_label,
-            self.inner
-                .tenant_timeline_describe(tenant_shard_id, timeline_id,)
-                .await
-        )
-    }
-    /* END_HADRON */
-
     pub(crate) async fn tenant_scan_remote_storage(
         &self,
         tenant_id: TenantId,

storage_controller/src/service.rs

@@ -32,7 +32,7 @@ use pageserver_api::controller_api::{
     ShardSchedulingPolicy, ShardsPreferredAzsRequest, ShardsPreferredAzsResponse,
     SkSchedulingPolicy, TenantCreateRequest, TenantCreateResponse, TenantCreateResponseShard,
     TenantDescribeResponse, TenantDescribeResponseShard, TenantLocateResponse, TenantPolicyRequest,
-    TenantShardMigrateRequest, TenantShardMigrateResponse, TenantTimelineDescribeResponse,
+    TenantShardMigrateRequest, TenantShardMigrateResponse,
 };
 use pageserver_api::models::{
     self, DetachBehavior, LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease,
@@ -5486,92 +5486,6 @@ impl Service {
         .ok_or_else(|| ApiError::NotFound(anyhow::anyhow!("Tenant {tenant_id} not found").into()))
     }
 
-    /* BEGIN_HADRON */
-    pub(crate) async fn tenant_timeline_describe(
-        &self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-    ) -> Result<TenantTimelineDescribeResponse, ApiError> {
-        self.tenant_remote_mutation(tenant_id, |locations| async move {
-            if locations.0.is_empty() {
-                return Err(ApiError::NotFound(
-                    anyhow::anyhow!("Tenant not found").into(),
-                ));
-            };
-
-            let locations: Vec<(TenantShardId, Node)> = locations
-                .0
-                .iter()
-                .map(|t| (*t.0, t.1.latest.node.clone()))
-                .collect();
-            let mut futs = FuturesUnordered::new();
-
-            for (shard_id, node) in locations {
-                futs.push({
-                    async move {
-                        let result = node
-                            .with_client_retries(
-                                |client| async move {
-                                    client
-                                        .tenant_timeline_describe(&shard_id, &timeline_id)
-                                        .await
-                                },
-                                &self.http_client,
-                                &self.config.pageserver_jwt_token,
-                                3,
-                                3,
-                                Duration::from_secs(30),
-                                &self.cancel,
-                            )
-                            .await;
-                        (result, shard_id, node.get_id())
-                    }
-                });
-            }
-
-            let mut results: Vec<TimelineInfo> = Vec::new();
-            while let Some((result, tenant_shard_id, node_id)) = futs.next().await {
-                match result {
-                    Some(Ok(timeline_info)) => results.push(timeline_info),
-                    Some(Err(e)) => {
-                        tracing::warn!(
-                            "Failed to describe tenant {} timeline {} for pageserver {}: {e}",
-                            tenant_shard_id,
-                            timeline_id,
-                            node_id,
-                        );
-                        return Err(ApiError::ResourceUnavailable(format!("{e}").into()));
-                    }
-                    None => return Err(ApiError::Cancelled),
-                }
-            }
-            let mut image_consistent_lsn: Option<Lsn> = Some(Lsn::MAX);
-            for timeline_info in &results {
-                if let Some(tline_image_consistent_lsn) = timeline_info.image_consistent_lsn {
-                    image_consistent_lsn = Some(std::cmp::min(
-                        image_consistent_lsn.unwrap(),
-                        tline_image_consistent_lsn,
-                    ));
-                } else {
-                    tracing::warn!(
-                        "Timeline {} on shard {} does not have image consistent lsn",
-                        timeline_info.timeline_id,
-                        timeline_info.tenant_id
-                    );
-                    image_consistent_lsn = None;
-                    break;
-                }
-            }
-
-            Ok(TenantTimelineDescribeResponse {
-                shards: results,
-                image_consistent_lsn,
-            })
-        })
-        .await?
-    }
-    /* END_HADRON */
-
     /// limit & offset are pagination parameters. Since we are walking an in-memory HashMap, `offset` does not
     /// avoid traversing data, it just avoid returning it. This is suitable for our purposes, since our in memory
     /// maps are small enough to traverse fast, our pagination is just to avoid serializing huge JSON responses

storage_controller/src/service/safekeeper_service.rs

@@ -25,8 +25,7 @@ use pageserver_api::models::{SafekeeperInfo, SafekeepersInfo, TimelineInfo};
 use safekeeper_api::PgVersionId;
 use safekeeper_api::membership::{self, MemberSet, SafekeeperGeneration};
 use safekeeper_api::models::{
-    PullTimelineRequest, TimelineLocateResponse, TimelineMembershipSwitchRequest,
-    TimelineMembershipSwitchResponse,
+    PullTimelineRequest, TimelineMembershipSwitchRequest, TimelineMembershipSwitchResponse,
 };
 use safekeeper_api::{INITIAL_TERM, Term};
 use safekeeper_client::mgmt_api;
@@ -38,6 +37,13 @@ use utils::lsn::Lsn;
 
 use super::Service;
 
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
+pub struct TimelineLocateResponse {
+    pub generation: SafekeeperGeneration,
+    pub sk_set: Vec<NodeId>,
+    pub new_sk_set: Option<Vec<NodeId>>,
+}
+
 impl Service {
     fn make_member_set(safekeepers: &[Safekeeper]) -> Result<MemberSet, ApiError> {
         let members = safekeepers

test_runner/fixtures/endpoint/http.py

@@ -115,6 +115,39 @@ class EndpointHttpClient(requests.Session):
         json: dict[str, str] = res.json()
         return json
 
+    def start_profiling_cpu(
+        self, sampling_frequency: int, timeout_seconds: int, archive: bool = False
+    ) -> tuple[int, bytes]:
+        url = f"http://localhost:{self.external_port}/profile/cpu"
+        params = {
+            "profiler": {"BccProfile": None},
+            "sampling_frequency": sampling_frequency,
+            "timeout_seconds": timeout_seconds,
+            "archive": archive,
+        }
+
+        res = self.post(
+            url,
+            json=params,
+            auth=self.auth,
+        )
+
+        return res.status_code, res.content
+
+    def stop_profiling_cpu(self) -> int:
+        url = f"http://localhost:{self.external_port}/profile/cpu"
+        res = self.delete(url, auth=self.auth)
+        return res.status_code
+
+    def get_profiling_cpu_status(self) -> bool:
+        """
+        Returns True if CPU profiling is currently running, False otherwise.
+        """
+        url = f"http://localhost:{self.external_port}/profile/cpu"
+        res = self.get(url, auth=self.auth)
+        res.raise_for_status()
+        return res.status_code == 200
+
     def database_schema(self, database: str):
         res = self.get(
             f"http://localhost:{self.external_port}/database_schema?database={urllib.parse.quote(database, safe='')}",

test_runner/fixtures/neon_fixtures.py

@@ -2342,20 +2342,6 @@ class NeonStorageController(MetricsGetter, LogUtils):
         response.raise_for_status()
         return response.json()
 
-    # HADRON
-    def tenant_timeline_describe(
-        self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-    ):
-        response = self.request(
-            "GET",
-            f"{self.api}/control/v1/tenant/{tenant_id}/timeline/{timeline_id}",
-            headers=self.headers(TokenScope.ADMIN),
-        )
-        response.raise_for_status()
-        return response.json()
-
     def nodes(self):
         """
         :return: list of {"id": ""}

test_runner/fixtures/utils.py

@@ -6,6 +6,7 @@ import json
 import os
 import re
 import subprocess
+import sys
 import tarfile
 import threading
 import time
@@ -717,6 +718,33 @@ def skip_in_debug_build(reason: str):
     )
 
 
+def run_only_on_linux_kernel_higher_than(version: float, reason: str):
+    """
+    Skip tests if the Linux kernel version is lower than the specified version.
+    The version is specified as a float, e.g. 5.4 for kernel.
+
+    Also skips if the host is not Linux.
+    """
+
+    should_skip = False
+    if sys.platform != "linux":
+        should_skip = True
+    else:
+        try:
+            kernel_version_list = os.uname()[2].split("-")[0].split(".")
+            kernel_version = float(f"{kernel_version_list[0]}.{kernel_version_list[1]}")
+            if kernel_version < version:
+                should_skip = True
+        except ValueError:
+            log.error(f"Failed to parse kernel version: {os.uname()[2]}")
+            should_skip = True
+
+    return pytest.mark.skipif(
+        should_skip,
+        reason=reason,
+    )
+
+
 def skip_on_ci(reason: str):
     # `CI` variable is always set to `true` on GitHub
     # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables

test_runner/regress/data/profile_pb2.py

@@ -0,0 +1,41 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# type: ignore
+# source: profile.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rprofile.proto\x12\x12perftools.profiles\"\xd5\x03\n\x07Profile\x12\x32\n\x0bsample_type\x18\x01 \x03(\x0b\x32\x1d.perftools.profiles.ValueType\x12*\n\x06sample\x18\x02 \x03(\x0b\x32\x1a.perftools.profiles.Sample\x12,\n\x07mapping\x18\x03 \x03(\x0b\x32\x1b.perftools.profiles.Mapping\x12.\n\x08location\x18\x04 \x03(\x0b\x32\x1c.perftools.profiles.Location\x12.\n\x08\x66unction\x18\x05 \x03(\x0b\x32\x1c.perftools.profiles.Function\x12\x14\n\x0cstring_table\x18\x06 \x03(\t\x12\x13\n\x0b\x64rop_frames\x18\x07 \x01(\x03\x12\x13\n\x0bkeep_frames\x18\x08 \x01(\x03\x12\x12\n\ntime_nanos\x18\t \x01(\x03\x12\x16\n\x0e\x64uration_nanos\x18\n \x01(\x03\x12\x32\n\x0bperiod_type\x18\x0b \x01(\x0b\x32\x1d.perftools.profiles.ValueType\x12\x0e\n\x06period\x18\x0c \x01(\x03\x12\x0f\n\x07\x63omment\x18\r \x03(\x03\x12\x1b\n\x13\x64\x65\x66\x61ult_sample_type\x18\x0e \x01(\x03\"\'\n\tValueType\x12\x0c\n\x04type\x18\x01 \x01(\x03\x12\x0c\n\x04unit\x18\x02 \x01(\x03\"V\n\x06Sample\x12\x13\n\x0blocation_id\x18\x01 \x03(\x04\x12\r\n\x05value\x18\x02 \x03(\x03\x12(\n\x05label\x18\x03 \x03(\x0b\x32\x19.perftools.profiles.Label\"@\n\x05Label\x12\x0b\n\x03key\x18\x01 \x01(\x03\x12\x0b\n\x03str\x18\x02 \x01(\x03\x12\x0b\n\x03num\x18\x03 \x01(\x03\x12\x10\n\x08num_unit\x18\x04 \x01(\x03\"\xdd\x01\n\x07Mapping\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x14\n\x0cmemory_start\x18\x02 \x01(\x04\x12\x14\n\x0cmemory_limit\x18\x03 \x01(\x04\x12\x13\n\x0b\x66ile_offset\x18\x04 \x01(\x04\x12\x10\n\x08\x66ilename\x18\x05 \x01(\x03\x12\x10\n\x08\x62uild_id\x18\x06 \x01(\x03\x12\x15\n\rhas_functions\x18\x07 \x01(\x08\x12\x15\n\rhas_filenames\x18\x08 \x01(\x08\x12\x18\n\x10has_line_numbers\x18\t \x01(\x08\x12\x19\n\x11has_inline_frames\x18\n \x01(\x08\"v\n\x08Location\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x12\n\nmapping_id\x18\x02 \x01(\x04\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x01(\x04\x12&\n\x04line\x18\x04 \x03(\x0b\x32\x18.perftools.profiles.Line\x12\x11\n\tis_folded\x18\x05 \x01(\x08\")\n\x04Line\x12\x13\n\x0b\x66unction_id\x18\x01 \x01(\x04\x12\x0c\n\x04line\x18\x02 \x01(\x03\"_\n\x08\x46unction\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\x03\x12\x13\n\x0bsystem_name\x18\x03 \x01(\x03\x12\x10\n\x08\x66ilename\x18\x04 \x01(\x03\x12\x12\n\nstart_line\x18\x05 \x01(\x03\x42-\n\x1d\x63om.google.perftools.profilesB\x0cProfileProtob\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'profile_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'\n\035com.google.perftools.profilesB\014ProfileProto'
+  _PROFILE._serialized_start=38
+  _PROFILE._serialized_end=507
+  _VALUETYPE._serialized_start=509
+  _VALUETYPE._serialized_end=548
+  _SAMPLE._serialized_start=550
+  _SAMPLE._serialized_end=636
+  _LABEL._serialized_start=638
+  _LABEL._serialized_end=702
+  _MAPPING._serialized_start=705
+  _MAPPING._serialized_end=926
+  _LOCATION._serialized_start=928
+  _LOCATION._serialized_end=1046
+  _LINE._serialized_start=1048
+  _LINE._serialized_end=1089
+  _FUNCTION._serialized_start=1091
+  _FUNCTION._serialized_end=1186
+# @@protoc_insertion_point(module_scope)

test_runner/regress/test_compaction.py

@@ -960,9 +960,9 @@ def get_layer_map(env, tenant_shard_id, timeline_id, ps_id):
     return image_layer_count, delta_layer_count
 
 
-def test_image_layer_force_creation_period(neon_env_builder: NeonEnvBuilder):
+def test_image_creation_timeout(neon_env_builder: NeonEnvBuilder):
     """
-    Tests that page server can force creating new images if image_layer_force_creation_period is enabled
+    Tests that page server can force creating new images if image creation timeout is enabled
     """
     # use large knobs to disable L0 compaction/image creation except for the force image creation
     tenant_conf = {
@@ -972,10 +972,10 @@ def test_image_layer_force_creation_period(neon_env_builder: NeonEnvBuilder):
         "checkpoint_distance": 10 * 1024,
         "checkpoint_timeout": "1s",
         "image_layer_force_creation_period": "1s",
-        "pitr_interval": "10s",
-        "gc_period": "1s",
-        "compaction_period": "1s",
-        "lsn_lease_length": "1s",
+        # The lsn for forced image layer creations is calculated once every 10 minutes.
+        # Hence, drive compaction manually such that the test doesn't compute it at the
+        # wrong time.
+        "compaction_period": "0s",
     }
 
     # consider every tenant large to run the image layer generation check more eagerly
@@ -1018,69 +1018,4 @@ def test_image_layer_force_creation_period(neon_env_builder: NeonEnvBuilder):
     )
 
 
-def test_image_consistent_lsn(neon_env_builder: NeonEnvBuilder):
-    """
-    Test the /v1/tenant/<tenant_id>/timeline/<timeline_id> endpoint and the computation of image_consistent_lsn
-    """
-    # use large knobs to disable L0 compaction/image creation except for the force image creation
-    tenant_conf = {
-        "compaction_threshold": "100",
-        "image_creation_threshold": "100",
-        "image_layer_creation_check_threshold": "1",
-        "checkpoint_distance": 10 * 1024,
-        "checkpoint_timeout": "1s",
-        "image_layer_force_creation_period": "1s",
-        "pitr_interval": "10s",
-        "gc_period": "1s",
-        "compaction_period": "1s",
-        "lsn_lease_length": "1s",
-    }
-
-    neon_env_builder.num_pageservers = 2
-    neon_env_builder.num_safekeepers = 1
-    env = neon_env_builder.init_start(
-        initial_tenant_conf=tenant_conf,
-        initial_tenant_shard_count=4,
-        initial_tenant_shard_stripe_size=1,
-    )
-
-    tenant_id = env.initial_tenant
-    timeline_id = env.initial_timeline
-
-    endpoint = env.endpoints.create_start("main")
-    endpoint.safe_psql("CREATE TABLE foo (id INTEGER, val text)")
-    for v in range(10):
-        endpoint.safe_psql(
-            f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))", log_query=False
-        )
-
-    response = env.storage_controller.tenant_timeline_describe(tenant_id, timeline_id)
-    shards = response["shards"]
-    for shard in shards:
-        assert shard["image_consistent_lsn"] is not None
-    image_consistent_lsn = response["image_consistent_lsn"]
-    assert image_consistent_lsn is not None
-
-    # do more writes and wait for image_consistent_lsn to advance
-    for v in range(100):
-        endpoint.safe_psql(
-            f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))", log_query=False
-        )
-
-    def check_image_consistent_lsn_advanced():
-        response = env.storage_controller.tenant_timeline_describe(tenant_id, timeline_id)
-        new_image_consistent_lsn = response["image_consistent_lsn"]
-        shards = response["shards"]
-        for shard in shards:
-            print(f"shard {shard['tenant_id']} image_consistent_lsn{shard['image_consistent_lsn']}")
-        assert new_image_consistent_lsn != image_consistent_lsn
-
-    wait_until(check_image_consistent_lsn_advanced)
-
-    endpoint.stop_and_destroy()
-
-    for ps in env.pageservers:
-        ps.allowed_errors.append(".*created delta file of size.*larger than double of target.*")
-
-
 # END_HADRON

test_runner/regress/test_compute_profiling.py

@@ -0,0 +1,322 @@
+from __future__ import annotations
+
+import gzip
+import io
+import threading
+import time
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from data.profile_pb2 import Profile  # type: ignore
+from fixtures.log_helper import log
+from fixtures.utils import run_only_on_default_postgres, run_only_on_linux_kernel_higher_than
+from google.protobuf.message import Message
+from requests import HTTPError
+
+if TYPE_CHECKING:
+    from fixtures.endpoint.http import EndpointHttpClient
+    from fixtures.neon_fixtures import NeonEnv
+
+LINUX_VERSION_REQUIRED = 6.0
+PG_REASON = "test doesn't use postgres in a manner that requires any specific version"
+LINUX_REASON = f"test requires linux {LINUX_VERSION_REQUIRED} for ebpfs"
+
+
+def _start_profiling_cpu(
+    client: EndpointHttpClient, event: threading.Event | None, archive: bool = False
+) -> Profile | None:
+    """
+    Start CPU profiling for the compute node.
+    """
+    log.info("Starting CPU profiling...")
+
+    if event is not None:
+        event.set()
+
+    status, response = client.start_profiling_cpu(100, 30, archive=archive)
+    match status:
+        case 200:
+            log.debug("CPU profiling finished")
+            profile: Any = Profile()
+            if archive:
+                log.debug("Unpacking gzipped profile data")
+                with gzip.GzipFile(fileobj=io.BytesIO(response)) as f:
+                    response = f.read()
+            else:
+                log.debug("Unpacking non-gzipped profile data")
+            Message.ParseFromString(profile, response)
+            return profile
+        case 204:
+            log.debug("CPU profiling was stopped")
+            raise HTTPError("Failed to finish CPU profiling: was stopped.")
+        case 409:
+            log.debug("CPU profiling is already in progress, cannot start again")
+            raise HTTPError("Failed to finish CPU profiling: profiling is already in progress.")
+        case 500:
+            response_str = response.decode("utf-8", errors="replace")
+            log.debug(
+                f"Failed to finish CPU profiling, was stopped or got an error: {status}: {response_str}"
+            )
+            raise HTTPError(f"Failed to finish CPU profiling, {status}: {response_str}")
+        case _:
+            log.error(f"Failed to start CPU profiling: {status}")
+            raise HTTPError(f"Failed to start CPU profiling: {status}")
+
+
+def _stop_profiling_cpu(client: EndpointHttpClient, event: threading.Event | None):
+    """
+    Stop CPU profiling for the compute node.
+    """
+    log.info("Manually stopping CPU profiling...")
+
+    if event is not None:
+        event.set()
+
+    status = client.stop_profiling_cpu()
+    match status:
+        case 200:
+            log.debug("CPU profiling stopped successfully")
+        case 412:
+            log.debug("CPU profiling is not running, nothing to do")
+        case _:
+            log.error(f"Failed to stop CPU profiling: {status}")
+            raise HTTPError(f"Failed to stop CPU profiling: {status}")
+    return status
+
+
+def _wait_till_profiling_starts(
+    http_client: EndpointHttpClient,
+    event: threading.Event | None,
+    repeat_delay_secs: float = 0.3,
+    timeout: int = 60,
+) -> bool:
+    """
+    Wait until CPU profiling starts.
+    """
+    log.info("Waiting for CPU profiling to start...")
+
+    end_time = time.time() + timeout
+
+    while not http_client.get_profiling_cpu_status():
+        time.sleep(repeat_delay_secs)
+
+        if end_time <= time.time():
+            log.error("Timeout while waiting for CPU profiling to start")
+            return False
+
+    log.info("CPU profiling has started successfully and is in progress")
+
+    if event is not None:
+        event.set()
+
+    return True
+
+
+def _wait_and_assert_cpu_profiling(http_client: EndpointHttpClient, event: threading.Event | None):
+    profile = _start_profiling_cpu(http_client, event)
+
+    if profile is None:
+        log.error("The received profiling data is malformed or empty.")
+        return
+
+    assert len(profile.sample) > 0, "No samples found in CPU profiling data"
+    assert len(profile.mapping) > 0, "No mappings found in CPU profiling data"
+    assert len(profile.location) > 0, "No locations found in CPU profiling data"
+    assert len(profile.function) > 0, "No functions found in CPU profiling data"
+    assert len(profile.string_table) > 0, "No string tables found in CPU profiling data"
+    strings = [
+        "PostgresMain",
+        "ServerLoop",
+        "BackgroundWorkerMain",
+        "pq_recvbuf",
+        "pq_getbyte",
+    ]
+
+    assert any(s in profile.string_table for s in strings), (
+        f"Expected at least one of {strings} in string table, but none found"
+    )
+
+
+@run_only_on_default_postgres(reason=PG_REASON)
+@run_only_on_linux_kernel_higher_than(version=LINUX_VERSION_REQUIRED, reason=LINUX_REASON)
+def test_compute_profiling_cpu_with_timeout(neon_simple_env: NeonEnv):
+    """
+    Test that CPU profiling works correctly with timeout.
+    """
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+    pg_conn = endpoint.connect()
+    pg_cur = pg_conn.cursor()
+    pg_cur.execute("create database profiling_test2")
+    http_client = endpoint.http_client()
+
+    def _wait_and_assert_cpu_profiling_local():
+        _wait_and_assert_cpu_profiling(http_client, None)
+
+    thread = threading.Thread(target=_wait_and_assert_cpu_profiling_local)
+    thread.start()
+
+    inserting_should_stop_event = threading.Event()
+
+    def insert_rows():
+        lfc_conn = endpoint.connect(dbname="profiling_test2")
+        lfc_cur = lfc_conn.cursor()
+        n_records = 0
+        lfc_cur.execute(
+            "create table t(pk integer primary key, payload text default repeat('?', 128))"
+        )
+        batch_size = 1000
+
+        log.info("Inserting rows")
+
+        while not inserting_should_stop_event.is_set():
+            n_records += batch_size
+            lfc_cur.execute(
+                f"insert into t (pk) values (generate_series({n_records - batch_size + 1},{batch_size}))"
+            )
+
+        log.info(f"Inserted {n_records} rows")
+
+    thread2 = threading.Thread(target=insert_rows)
+
+    assert _wait_till_profiling_starts(http_client, None)
+    time.sleep(4)  # Give some time for the profiling to start
+    thread2.start()
+
+    thread.join(timeout=60)
+    inserting_should_stop_event.set()  # Stop the insertion thread
+    thread2.join(timeout=60)
+
+
+@run_only_on_default_postgres(reason=PG_REASON)
+@run_only_on_linux_kernel_higher_than(version=LINUX_VERSION_REQUIRED, reason=LINUX_REASON)
+def test_compute_profiling_cpu_with_archiving_the_response(neon_simple_env: NeonEnv):
+    """
+    Test that CPU profiling works correctly with archiving the data.
+    """
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+    http_client = endpoint.http_client()
+
+    assert _start_profiling_cpu(http_client, None, archive=True) is not None, (
+        "Failed to start and finish CPU profiling with archiving enabled"
+    )
+
+
+@run_only_on_default_postgres(reason=PG_REASON)
+@run_only_on_linux_kernel_higher_than(version=LINUX_VERSION_REQUIRED, reason=LINUX_REASON)
+def test_compute_profiling_cpu_start_and_stop(neon_simple_env: NeonEnv):
+    """
+    Test that CPU profiling can be started and stopped correctly.
+    """
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+    http_client = endpoint.http_client()
+
+    def _wait_and_assert_cpu_profiling():
+        # Should raise as the profiling will be stopped.
+        with pytest.raises(HTTPError) as _:
+            _start_profiling_cpu(http_client, None)
+
+    thread = threading.Thread(target=_wait_and_assert_cpu_profiling)
+    thread.start()
+
+    assert _wait_till_profiling_starts(http_client, None)
+    _stop_profiling_cpu(http_client, None)
+
+    # Should raise as the profiling is already stopped.
+    assert _stop_profiling_cpu(http_client, None) == 412
+
+    thread.join(timeout=60)
+
+
+@run_only_on_default_postgres(reason=PG_REASON)
+@run_only_on_linux_kernel_higher_than(version=LINUX_VERSION_REQUIRED, reason=LINUX_REASON)
+def test_compute_profiling_cpu_conflict(neon_simple_env: NeonEnv):
+    """
+    Test that CPU profiling can be started once and the second time
+    it will throw an error as it is already running.
+    """
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+    pg_conn = endpoint.connect()
+    pg_cur = pg_conn.cursor()
+    pg_cur.execute("create database profiling_test")
+    http_client = endpoint.http_client()
+
+    def _wait_and_assert_cpu_profiling_local():
+        _wait_and_assert_cpu_profiling(http_client, None)
+
+    thread = threading.Thread(target=_wait_and_assert_cpu_profiling_local)
+    thread.start()
+
+    assert _wait_till_profiling_starts(http_client, None)
+
+    inserting_should_stop_event = threading.Event()
+
+    def insert_rows():
+        lfc_conn = endpoint.connect(dbname="profiling_test")
+        lfc_cur = lfc_conn.cursor()
+        n_records = 0
+        lfc_cur.execute(
+            "create table t(pk integer primary key, payload text default repeat('?', 128))"
+        )
+        batch_size = 1000
+
+        log.info("Inserting rows")
+
+        while not inserting_should_stop_event.is_set():
+            n_records += batch_size
+            lfc_cur.execute(
+                f"insert into t (pk) values (generate_series({n_records - batch_size + 1},{batch_size}))"
+            )
+
+        log.info(f"Inserted {n_records} rows")
+
+    thread2 = threading.Thread(target=insert_rows)
+    thread2.start()
+
+    # Should raise as the profiling is already in progress.
+    with pytest.raises(HTTPError) as _:
+        _start_profiling_cpu(http_client, None)
+
+    inserting_should_stop_event.set()  # Stop the insertion thread
+
+    # The profiling should still be running and finish normally.
+    thread.join(timeout=600)
+    thread2.join(timeout=600)
+
+
+@run_only_on_default_postgres(reason=PG_REASON)
+@run_only_on_linux_kernel_higher_than(version=LINUX_VERSION_REQUIRED, reason=LINUX_REASON)
+def test_compute_profiling_cpu_stop_when_not_running(neon_simple_env: NeonEnv):
+    """
+    Test that CPU profiling throws the expected error when is attempted
+    to be stopped when it hasn't be running.
+    """
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+    http_client = endpoint.http_client()
+
+    for _ in range(3):
+        status = _stop_profiling_cpu(http_client, None)
+        assert status == 412
+
+
+@run_only_on_default_postgres(reason=PG_REASON)
+@run_only_on_linux_kernel_higher_than(version=LINUX_VERSION_REQUIRED, reason=LINUX_REASON)
+def test_compute_profiling_cpu_start_arguments_validation_works(neon_simple_env: NeonEnv):
+    """
+    Test that CPU profiling start request properly validated the
+    arguments and throws the expected error (bad request).
+    """
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+    http_client = endpoint.http_client()
+
+    for sampling_frequency in [-1, 0, 1000000]:
+        status, _ = http_client.start_profiling_cpu(sampling_frequency, 5)
+        assert status == 422
+    for timeout_seconds in [-1, 0, 1000000]:
+        status, _ = http_client.start_profiling_cpu(5, timeout_seconds)
+        assert status == 422

workspace_hack/Cargo.toml

@@ -28,6 +28,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd
 clap = { version = "4", features = ["derive", "env", "string"] }
 clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] }
 const-oid = { version = "0.9", default-features = false, features = ["db", "std"] }
+criterion = { version = "0.5", features = ["html_reports"] }
 crypto-bigint = { version = "0.5", features = ["generic-array", "zeroize"] }
 der = { version = "0.7", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] }
 deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] }
@@ -54,7 +55,8 @@ hmac = { version = "0.12", default-features = false, features = ["reset"] }
 hyper-582f2526e08bb6a0 = { package = "hyper", version = "0.14", features = ["client", "http1", "http2", "runtime", "server", "stream"] }
 hyper-dff4ba8e3ae991db = { package = "hyper", version = "1", features = ["full"] }
 hyper-util = { version = "0.1", features = ["client-legacy", "server-auto", "service"] }
-indexmap = { version = "2", features = ["serde"] }
+indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] }
+indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] }
 itertools = { version = "0.12" }
 lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] }
 libc = { version = "0.2", features = ["extra_traits", "use_std"] }
@@ -70,9 +72,7 @@ num-integer = { version = "0.1", features = ["i128"] }
 num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] }
 num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] }
 num-traits = { version = "0.2", features = ["i128", "libm"] }
-once_cell = { version = "1" }
 p256 = { version = "0.13", features = ["jwk"] }
-paracord = { version = "0.1", features = ["serde"] }
 parquet = { version = "53", default-features = false, features = ["zstd"] }
 prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] }
 rand = { version = "0.8", features = ["small_rng"] }
@@ -90,6 +90,7 @@ serde_json = { version = "1", features = ["alloc", "raw_value"] }
 sha2 = { version = "0.10", features = ["asm", "oid"] }
 signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] }
 smallvec = { version = "1", default-features = false, features = ["const_new", "write"] }
+spin = { version = "0.9" }
 spki = { version = "0.7", default-features = false, features = ["pem", "std"] }
 stable_deref_trait = { version = "1" }
 subtle = { version = "2" }
@@ -102,6 +103,7 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["loggin
 tokio-stream = { version = "0.1", features = ["net"] }
 tokio-util = { version = "0.7", features = ["codec", "compat", "io-util", "rt"] }
 toml_edit = { version = "0.22", features = ["serde"] }
+tonic = { version = "0.12", default-features = false, features = ["codegen", "prost", "transport"] }
 tower = { version = "0.5", default-features = false, features = ["balance", "buffer", "limit", "log"] }
 tracing = { version = "0.1", features = ["log"] }
 tracing-core = { version = "0.1" }
@@ -121,13 +123,15 @@ anyhow = { version = "1", features = ["backtrace"] }
 bytes = { version = "1", features = ["serde"] }
 cc = { version = "1", default-features = false, features = ["parallel"] }
 chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] }
+clang-sys = { version = "1", default-features = false, features = ["clang_11_0", "runtime"] }
 clap = { version = "4", features = ["derive", "env", "string"] }
 clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] }
 either = { version = "1" }
 getrandom = { version = "0.2", default-features = false, features = ["std"] }
 half = { version = "2", default-features = false, features = ["num-traits"] }
 hashbrown = { version = "0.14", features = ["raw"] }
-indexmap = { version = "2", features = ["serde"] }
+indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] }
+indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] }
 itertools = { version = "0.12" }
 libc = { version = "0.2", features = ["extra_traits", "use_std"] }
 log = { version = "0.4", default-features = false, features = ["std"] }
@@ -140,7 +144,6 @@ num-integer = { version = "0.1", features = ["i128"] }
 num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] }
 num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] }
 num-traits = { version = "0.2", features = ["i128", "libm"] }
-once_cell = { version = "1" }
 parquet = { version = "53", default-features = false, features = ["zstd"] }
 prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] }
 proc-macro2 = { version = "1" }