pageserver: API for invoking page trace

pageserver: add PageTrace machinery
Merge pull request #10338 from neondatabase/rc/release/2025-01-10
2026-06-01 12:30:38 +00:00 · 2025-01-14 18:04:59 +00:00 · 2025-01-14 18:04:59 +00:00 · 2025-01-10 19:21:43 +00:00 · 2025-01-10 06:02:00 +00:00 · 2025-01-03 20:32:37 +01:00
28 changed files with 286 additions and 393 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1653,20 +1653,6 @@ dependencies = [
 "parking_lot_core 0.9.8",
 ]

-[[package]]
-name = "dashmap"
-version = "6.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
- "hashbrown 0.14.5",
- "lock_api",
- "once_cell",
- "parking_lot_core 0.9.8",
-]
-
 [[package]]
 name = "data-encoding"
 version = "2.4.0"
@@ -1966,15 +1952,6 @@ dependencies = [
 "syn 2.0.90",
 ]

-[[package]]
-name = "env_filter"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
-dependencies = [
- "log",
-]
-
 [[package]]
 name = "env_logger"
 version = "0.10.2"
@@ -1988,16 +1965,6 @@ dependencies = [
 "termcolor",
 ]

-[[package]]
-name = "env_logger"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d"
-dependencies = [
- "env_filter",
- "log",
-]
-
 [[package]]
 name = "equator"
 version = "0.2.2"
@@ -2981,28 +2948,6 @@ dependencies = [
 "str_stack",
 ]

-[[package]]
-name = "inferno"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a5d75fee4d36809e6b021e4b96b686e763d365ffdb03af2bd00786353f84fe"
-dependencies = [
- "ahash",
- "clap",
- "crossbeam-channel",
- "crossbeam-utils",
- "dashmap 6.1.0",
- "env_logger 0.11.2",
- "indexmap 2.0.1",
- "itoa",
- "log",
- "num-format",
- "once_cell",
- "quick-xml 0.37.1",
- "rgb",
- "str_stack",
-]
-
 [[package]]
 name = "inotify"
 version = "0.9.6"
@@ -3210,7 +3155,7 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2"
 dependencies = [
- "dashmap 5.5.0",
+ "dashmap",
 "hashbrown 0.13.2",
 ]

@@ -3748,23 +3693,23 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"

 [[package]]
 name = "opentelemetry"
-version = "0.27.1"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7"
+checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17"
 dependencies = [
 "futures-core",
 "futures-sink",
 "js-sys",
+ "once_cell",
 "pin-project-lite",
 "thiserror",
- "tracing",
 ]

 [[package]]
 name = "opentelemetry-http"
-version = "0.27.0"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80"
+checksum = "6351496aeaa49d7c267fb480678d85d1cd30c5edb20b497c48c56f62a8c14b99"
 dependencies = [
 "async-trait",
 "bytes",
@@ -3775,9 +3720,9 @@ dependencies = [

 [[package]]
 name = "opentelemetry-otlp"
-version = "0.27.0"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76"
+checksum = "29e1f9c8b032d4f635c730c0efcf731d5e2530ea13fa8bef7939ddc8420696bd"
 dependencies = [
 "async-trait",
 "futures-core",
@@ -3793,9 +3738,9 @@ dependencies = [

 [[package]]
 name = "opentelemetry-proto"
-version = "0.27.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6"
+checksum = "c9d3968ce3aefdcca5c27e3c4ea4391b37547726a70893aab52d3de95d5f8b34"
 dependencies = [
 "opentelemetry",
 "opentelemetry_sdk",
@@ -3805,21 +3750,22 @@ dependencies = [

 [[package]]
 name = "opentelemetry-semantic-conventions"
-version = "0.27.0"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
+checksum = "db945c1eaea8ac6a9677185357480d215bb6999faa9f691d0c4d4d641eab7a09"

 [[package]]
 name = "opentelemetry_sdk"
-version = "0.27.1"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8"
+checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3"
 dependencies = [
 "async-trait",
 "futures-channel",
 "futures-executor",
 "futures-util",
 "glob",
+ "once_cell",
 "opentelemetry",
 "percent-encoding",
 "rand 0.8.5",
@@ -3827,7 +3773,6 @@ dependencies = [
 "thiserror",
 "tokio",
 "tokio-stream",
- "tracing",
 ]

 [[package]]
@@ -3951,6 +3896,7 @@ dependencies = [
 "arc-swap",
 "async-compression",
 "async-stream",
+ "bincode",
 "bit_field",
 "byteorder",
 "bytes",
@@ -4476,7 +4422,7 @@ dependencies = [
 "bytes",
 "crc32c",
 "criterion",
- "env_logger 0.10.2",
+ "env_logger",
 "log",
 "memoffset 0.9.0",
 "once_cell",
@@ -4517,7 +4463,7 @@ dependencies = [
 "cfg-if",
 "criterion",
 "findshlibs",
- "inferno 0.11.21",
+ "inferno",
 "libc",
 "log",
 "nix 0.26.4",
@@ -4743,9 +4689,9 @@ dependencies = [
 "clap",
 "compute_api",
 "consumption_metrics",
- "dashmap 5.5.0",
+ "dashmap",
 "ecdsa 0.16.9",
- "env_logger 0.10.2",
+ "env_logger",
 "fallible-iterator",
 "flate2",
 "framed-websockets",
@@ -4852,15 +4798,6 @@ dependencies = [
 "serde",
 ]

-[[package]]
-name = "quick-xml"
-version = "0.37.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.37"
@@ -5245,9 +5182,9 @@ dependencies = [

 [[package]]
 name = "reqwest-tracing"
-version = "0.5.5"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73e6153390585f6961341b50e5a1931d6be6dee4292283635903c26ef9d980d2"
+checksum = "ff82cf5730a1311fb9413b0bc2b8e743e0157cd73f010ab4ec374a923873b6a2"
 dependencies = [
 "anyhow",
 "async-trait",
@@ -7112,9 +7049,9 @@ dependencies = [

 [[package]]
 name = "tracing-opentelemetry"
-version = "0.28.0"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053"
+checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b"
 dependencies = [
 "js-sys",
 "once_cell",
@@ -7383,7 +7320,6 @@ dependencies = [
 "hex-literal",
 "humantime",
 "hyper 0.14.30",
- "inferno 0.12.0",
 "itertools 0.10.5",
 "jemalloc_pprof",
 "jsonwebtoken",
@@ -7487,7 +7423,7 @@ dependencies = [
 "anyhow",
 "camino-tempfile",
 "clap",
- "env_logger 0.10.2",
+ "env_logger",
 "log",
 "postgres",
 "postgres_ffi",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -110,7 +110,6 @@ hyper-util = "0.1"
 tokio-tungstenite = "0.21.0"
 indexmap = "2"
 indoc = "2"
-inferno = "0.12.0"
 ipnet = "2.10.0"
 itertools = "0.10"
 itoa = "1.0.11"
@@ -127,10 +126,10 @@ notify = "6.0.0"
 num_cpus = "1.15"
 num-traits = "0.2.15"
 once_cell = "1.13"
-opentelemetry = "0.27"
-opentelemetry_sdk = "0.27"
-opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] }
-opentelemetry-semantic-conventions = "0.27"
+opentelemetry = "0.26"
+opentelemetry_sdk = "0.26"
+opentelemetry-otlp = { version = "0.26", default-features=false, features = ["http-proto", "trace", "http", "reqwest-client"] }
+opentelemetry-semantic-conventions = "0.26"
 parking_lot = "0.12"
 parquet = { version = "53", default-features = false, features = ["zstd"] }
 parquet_derive = "53"
@@ -144,7 +143,7 @@ rand = "0.8"
 redis = { version = "0.25.2", features = ["tokio-rustls-comp", "keep-alive"] }
 regex = "1.10.2"
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
-reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_27"] }
+reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_26"] }
 reqwest-middleware = "0.4"
 reqwest-retry = "0.7"
 routerify = "3"
@@ -193,7 +192,7 @@ tower-http = { version = "0.6.2", features = ["request-id", "trace"] }
 tower-service = "0.3.3"
 tracing = "0.1"
 tracing-error = "0.2"
-tracing-opentelemetry = "0.28"
+tracing-opentelemetry = "0.27"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
 try-lock = "0.2.5"
 twox-hash = { version = "1.6.3", default-features = false }
--- a/compute/compute-node.Dockerfile
+++ b/compute/compute-node.Dockerfile
@@ -1167,13 +1167,22 @@ FROM rust-extensions-build AS pg-mooncake-build
 ARG PG_VERSION
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/

+# The topmost commit in the `neon` branch at the time of writing this
+# https://github.com/Mooncake-Labs/pg_mooncake/commits/neon/
+# https://github.com/Mooncake-Labs/pg_mooncake/commit/077c92c452bb6896a7b7776ee95f039984f076af
+ENV PG_MOONCAKE_VERSION=077c92c452bb6896a7b7776ee95f039984f076af
 ENV PATH="/usr/local/pgsql/bin/:$PATH"

-RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.0/pg_mooncake-0.1.0.tar.gz -O pg_mooncake.tar.gz && \
-    echo "eafd059b77f541f11525eb8affcd66a176968cbd8fe7c0d436e733f2aa4da59f pg_mooncake.tar.gz" | sha256sum --check && \
-    mkdir pg_mooncake-src && cd pg_mooncake-src && tar xzf ../pg_mooncake.tar.gz --strip-components=1 -C . && \
-    make release -j $(getconf _NPROCESSORS_ONLN) && \
-    make install -j $(getconf _NPROCESSORS_ONLN) && \
+RUN case "${PG_VERSION}" in \
+        'v14') \
+            echo "pg_mooncake is not supported on Postgres ${PG_VERSION}" && exit 0;; \
+    esac && \
+    git clone --depth 1 --branch neon https://github.com/Mooncake-Labs/pg_mooncake.git pg_mooncake-src && \
+    cd pg_mooncake-src && \
+    git checkout "${PG_MOONCAKE_VERSION}" && \
+    git submodule update --init --depth 1 --recursive && \
+    make BUILD_TYPE=release -j $(getconf _NPROCESSORS_ONLN) && \
+    make BUILD_TYPE=release -j $(getconf _NPROCESSORS_ONLN) install && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_mooncake.control

 #########################################################################################
--- a/compute_tools/src/bin/compute_ctl.rs
+++ b/compute_tools/src/bin/compute_ctl.rs
@@ -111,6 +111,11 @@ fn main() -> Result<()> {
 fn init() -> Result<(String, clap::ArgMatches)> {
    init_tracing_and_logging(DEFAULT_LOG_LEVEL)?;

+    opentelemetry::global::set_error_handler(|err| {
+        tracing::info!("OpenTelemetry error: {err}");
+    })
+    .expect("global error handler lock poisoned");
+
    let mut signals = Signals::new([SIGINT, SIGTERM, SIGQUIT])?;
    thread::spawn(move || {
        for sig in signals.forever() {
--- a/libs/tracing-utils/src/lib.rs
+++ b/libs/tracing-utils/src/lib.rs
@@ -38,6 +38,7 @@ pub mod http;

 use opentelemetry::trace::TracerProvider;
 use opentelemetry::KeyValue;
+use opentelemetry_sdk::Resource;
 use tracing::Subscriber;
 use tracing_subscriber::registry::LookupSpan;
 use tracing_subscriber::Layer;
@@ -120,10 +121,7 @@ where
    S: Subscriber + for<'span> LookupSpan<'span>,
 {
    // Sets up exporter from the OTEL_EXPORTER_* environment variables.
-    let exporter = opentelemetry_otlp::SpanExporter::builder()
-        .with_http()
-        .build()
-        .expect("could not initialize opentelemetry exporter");
+    let exporter = opentelemetry_otlp::new_exporter().http();

    // TODO: opentelemetry::global::set_error_handler() with custom handler that
    //       bypasses default tracing layers, but logs regular looking log
@@ -134,13 +132,17 @@ where
        opentelemetry_sdk::propagation::TraceContextPropagator::new(),
    );

-    let tracer = opentelemetry_sdk::trace::TracerProvider::builder()
-        .with_batch_exporter(exporter, opentelemetry_sdk::runtime::Tokio)
-        .with_resource(opentelemetry_sdk::Resource::new(vec![KeyValue::new(
-            opentelemetry_semantic_conventions::resource::SERVICE_NAME,
-            service_name,
-        )]))
-        .build()
+    let tracer = opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_exporter(exporter)
+        .with_trace_config(opentelemetry_sdk::trace::Config::default().with_resource(
+            Resource::new(vec![KeyValue::new(
+                opentelemetry_semantic_conventions::resource::SERVICE_NAME,
+                service_name,
+            )]),
+        ))
+        .install_batch(opentelemetry_sdk::runtime::Tokio)
+        .expect("could not initialize opentelemetry exporter")
        .tracer("global");

    tracing_opentelemetry::layer().with_tracer(tracer)
--- a/libs/utils/Cargo.toml
+++ b/libs/utils/Cargo.toml
@@ -26,7 +26,6 @@ git-version.workspace = true
 hex = { workspace = true, features = ["serde"] }
 humantime.workspace = true
 hyper0 = { workspace = true, features = ["full"] }
-inferno.workspace = true
 itertools.workspace = true
 fail.workspace = true
 futures = { workspace = true }
--- a/libs/utils/src/http/endpoint.rs
+++ b/libs/utils/src/http/endpoint.rs
@@ -417,7 +417,6 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
    enum Format {
        Jemalloc,
        Pprof,
-        Svg,
    }

    // Parameters.
@@ -425,24 +424,9 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
        None => Format::Pprof,
        Some("jemalloc") => Format::Jemalloc,
        Some("pprof") => Format::Pprof,
-        Some("svg") => Format::Svg,
        Some(format) => return Err(ApiError::BadRequest(anyhow!("invalid format {format}"))),
    };

-    // Functions and mappings to strip when symbolizing pprof profiles. If true,
-    // also remove child frames.
-    static STRIP_FUNCTIONS: Lazy<Vec<(Regex, bool)>> = Lazy::new(|| {
-        vec![
-            (Regex::new("^__rust").unwrap(), false),
-            (Regex::new("^_start$").unwrap(), false),
-            (Regex::new("^irallocx_prof").unwrap(), true),
-            (Regex::new("^prof_alloc_prep").unwrap(), true),
-            (Regex::new("^std::rt::lang_start").unwrap(), false),
-            (Regex::new("^std::sys::backtrace::__rust").unwrap(), false),
-        ]
-    });
-    const STRIP_MAPPINGS: &[&str] = &["libc", "libgcc", "pthread", "vdso"];
-
    // Obtain profiler handle.
    let mut prof_ctl = jemalloc_pprof::PROF_CTL
        .as_ref()
@@ -480,9 +464,24 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
                // Symbolize the profile.
                // TODO: consider moving this upstream to jemalloc_pprof and avoiding the
                // serialization roundtrip.
+                static STRIP_FUNCTIONS: Lazy<Vec<(Regex, bool)>> = Lazy::new(|| {
+                    // Functions to strip from profiles. If true, also remove child frames.
+                    vec![
+                        (Regex::new("^__rust").unwrap(), false),
+                        (Regex::new("^_start$").unwrap(), false),
+                        (Regex::new("^irallocx_prof").unwrap(), true),
+                        (Regex::new("^prof_alloc_prep").unwrap(), true),
+                        (Regex::new("^std::rt::lang_start").unwrap(), false),
+                        (Regex::new("^std::sys::backtrace::__rust").unwrap(), false),
+                    ]
+                });
                let profile = pprof::decode(&bytes)?;
                let profile = pprof::symbolize(profile)?;
-                let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
+                let profile = pprof::strip_locations(
+                    profile,
+                    &["libc", "libgcc", "pthread", "vdso"],
+                    &STRIP_FUNCTIONS,
+                );
                pprof::encode(&profile)
            })
            .await
@@ -495,27 +494,6 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
                .body(Body::from(data))
                .map_err(|err| ApiError::InternalServerError(err.into()))
        }
-
-        Format::Svg => {
-            let body = tokio::task::spawn_blocking(move || {
-                let bytes = prof_ctl.dump_pprof()?;
-                let profile = pprof::decode(&bytes)?;
-                let profile = pprof::symbolize(profile)?;
-                let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
-                let mut opts = inferno::flamegraph::Options::default();
-                opts.title = "Heap inuse".to_string();
-                opts.count_name = "bytes".to_string();
-                pprof::flamegraph(profile, &mut opts)
-            })
-            .await
-            .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
-            .map_err(ApiError::InternalServerError)?;
-            Response::builder()
-                .status(200)
-                .header(CONTENT_TYPE, "image/svg+xml")
-                .body(Body::from(body))
-                .map_err(|err| ApiError::InternalServerError(err.into()))
-        }
    }
 }

--- a/libs/utils/src/pprof.rs
+++ b/libs/utils/src/pprof.rs
@@ -1,9 +1,8 @@
-use anyhow::bail;
 use flate2::write::{GzDecoder, GzEncoder};
 use flate2::Compression;
 use itertools::Itertools as _;
 use once_cell::sync::Lazy;
-use pprof::protos::{Function, Line, Location, Message as _, Profile};
+use pprof::protos::{Function, Line, Message as _, Profile};
 use regex::Regex;

 use std::borrow::Cow;
@@ -189,59 +188,3 @@ pub fn strip_locations(

    profile
 }
-
-/// Generates an SVG flamegraph from a symbolized pprof profile.
-pub fn flamegraph(
-    profile: Profile,
-    opts: &mut inferno::flamegraph::Options,
-) -> anyhow::Result<Vec<u8>> {
-    if profile.mapping.iter().any(|m| !m.has_functions) {
-        bail!("profile not symbolized");
-    }
-
-    // Index locations, functions, and strings.
-    let locations: HashMap<u64, Location> =
-        profile.location.into_iter().map(|l| (l.id, l)).collect();
-    let functions: HashMap<u64, Function> =
-        profile.function.into_iter().map(|f| (f.id, f)).collect();
-    let strings = profile.string_table;
-
-    // Resolve stacks as function names, and sum sample values per stack. Also reverse the stack,
-    // since inferno expects it bottom-up.
-    let mut stacks: HashMap<Vec<&str>, i64> = HashMap::new();
-    for sample in profile.sample {
-        let mut stack = Vec::with_capacity(sample.location_id.len());
-        for location in sample.location_id.into_iter().rev() {
-            let Some(location) = locations.get(&location) else {
-                bail!("missing location {location}");
-            };
-            for line in location.line.iter().rev() {
-                let Some(function) = functions.get(&line.function_id) else {
-                    bail!("missing function {}", line.function_id);
-                };
-                let Some(name) = strings.get(function.name as usize) else {
-                    bail!("missing string {}", function.name);
-                };
-                stack.push(name.as_str());
-            }
-        }
-        let Some(&value) = sample.value.first() else {
-            bail!("missing value");
-        };
-        *stacks.entry(stack).or_default() += value;
-    }
-
-    // Construct stack lines for inferno.
-    let lines = stacks
-        .into_iter()
-        .map(|(stack, value)| (stack.into_iter().join(";"), value))
-        .map(|(stack, value)| format!("{stack} {value}"))
-        .sorted()
-        .collect_vec();
-
-    // Construct the flamegraph.
-    let mut bytes = Vec::new();
-    let lines = lines.iter().map(|line| line.as_str());
-    inferno::flamegraph::from_lines(opts, lines, &mut bytes)?;
-    Ok(bytes)
-}
--- a/libs/utils/src/sync/spsc_fold.rs
+++ b/libs/utils/src/sync/spsc_fold.rs
@@ -96,11 +96,7 @@ impl<T: Send> Sender<T> {
                    }
                }
                State::SenderWaitsForReceiverToConsume(_data) => {
-                    // SAFETY: send is single threaded due to `&mut self` requirement,
-                    // therefore register is not concurrent.
-                    unsafe {
-                        self.state.wake_sender.register(cx.waker());
-                    }
+                    // Really, we shouldn't be polled until receiver has consumed and wakes us.
                    Poll::Pending
                }
                State::ReceiverGone => Poll::Ready(Err(SendError::ReceiverGone)),
@@ -453,38 +449,4 @@ mod tests {
        let err = recv_task.await.unwrap().expect_err("should error");
        assert!(matches!(err, RecvError::SenderGone));
    }
-
-    #[tokio::test(start_paused = true)]
-    async fn test_receiver_drop_while_waiting_for_receiver_to_consume_unblocks_sender() {
-        let (mut sender, receiver) = channel();
-
-        let state = receiver.state.clone();
-
-        sender.send((), |_, _| unreachable!()).await.unwrap();
-
-        assert!(matches!(&*state.value.lock().unwrap(), &State::HasData(_)));
-
-        let unmergeable = sender.send((), |_, _| Err(()));
-        let mut unmergeable = std::pin::pin!(unmergeable);
-        tokio::select! {
-            _ = tokio::time::sleep(FOREVER) => {},
-            _ = &mut unmergeable => {
-                panic!("unmergeable should not complete");
-            },
-        }
-
-        assert!(matches!(
-            &*state.value.lock().unwrap(),
-            &State::SenderWaitsForReceiverToConsume(_)
-        ));
-
-        drop(receiver);
-
-        assert!(matches!(
-            &*state.value.lock().unwrap(),
-            &State::ReceiverGone
-        ));
-
-        unmergeable.await.unwrap_err();
-    }
 }
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -16,6 +16,7 @@ arc-swap.workspace = true
 async-compression.workspace = true
 async-stream.workspace = true
 bit_field.workspace = true
+bincode.workspace = true
 byteorder.workspace = true
 bytes.workspace = true
 camino.workspace = true
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -53,10 +53,12 @@ project_build_tag!(BUILD_TAG);
 #[global_allocator]
 static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

-/// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20).
-#[allow(non_upper_case_globals)]
-#[export_name = "malloc_conf"]
-pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0";
+// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20).
+// TODO: disabled because concurrent CPU profiles cause seg faults. See:
+// https://github.com/neondatabase/neon/issues/10225.
+//#[allow(non_upper_case_globals)]
+//#[export_name = "malloc_conf"]
+//pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0";

 const PID_FILE_NAME: &str = "pageserver.pid";

--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -7,6 +7,7 @@ use std::collections::HashMap;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
+use std::time::Instant;

 use anyhow::{anyhow, Context, Result};
 use enumset::EnumSet;
@@ -90,6 +91,7 @@ use crate::tenant::timeline::CompactFlags;
 use crate::tenant::timeline::CompactOptions;
 use crate::tenant::timeline::CompactRequest;
 use crate::tenant::timeline::CompactionError;
+use crate::tenant::timeline::PageTrace;
 use crate::tenant::timeline::Timeline;
 use crate::tenant::GetTimelineError;
 use crate::tenant::OffloadedTimeline;
@@ -1521,6 +1523,60 @@ async fn timeline_gc_unblocking_handler(
    block_or_unblock_gc(request, false).await
 }

+async fn timeline_page_trace_handler(
+    request: Request<Body>,
+    _cancel: CancellationToken,
+) -> Result<Response<Body>, ApiError> {
+    let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
+    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
+    let state = get_state(&request);
+
+    let size_limit =
+        parse_query_param::<_, u64>(&request, "size_limit_bytes")?.unwrap_or(1024 * 1024);
+    let time_limit_secs = parse_query_param::<_, u64>(&request, "time_limit_secs")?.unwrap_or(5);
+
+    // Convert size limit to event limit based on known serialized size of an event
+    let event_limit = size_limit / 32;
+
+    let timeline =
+        active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
+            .await?;
+
+    let (page_trace, mut trace_rx) = PageTrace::new(event_limit);
+    timeline.page_trace.store(Arc::new(Some(page_trace)));
+
+    let mut buffer = Vec::with_capacity(size_limit as usize);
+
+    let deadline = Instant::now() + Duration::from_secs(time_limit_secs);
+
+    loop {
+        let timeout = deadline.saturating_duration_since(Instant::now());
+        tokio::select! {
+            event = trace_rx.recv() => {
+                buffer.extend(bincode::serialize(&event).unwrap());
+
+                if buffer.len() >= size_limit as usize {
+                    // Size threshold reached
+                    break;
+                }
+            }
+            _ = tokio::time::sleep(timeout) => {
+                // Time threshold reached
+                break;
+            }
+        }
+    }
+
+    // Above code is infallible, so we guarantee to switch the trace off when done
+    timeline.page_trace.store(Arc::new(None));
+
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/octet-stream")
+        .body(hyper::Body::from(buffer))
+        .unwrap())
+}
+
 /// Adding a block is `POST ../block_gc`, removing a block is `POST ../unblock_gc`.
 ///
 /// Both are technically unsafe because they might fire off index uploads, thus they are POST.
@@ -3487,6 +3543,10 @@ pub fn make_router(
            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/unblock_gc",
            |r| api_handler(r, timeline_gc_unblocking_handler),
        )
+        .post(
+            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/page_trace",
+            |r| api_handler(r, timeline_page_trace_handler),
+        )
        .post("/v1/tenant/:tenant_shard_id/heatmap_upload", |r| {
            api_handler(r, secondary_upload_handler)
        })
--- a/pageserver/src/page_service.rs
+++ b/pageserver/src/page_service.rs
@@ -61,7 +61,7 @@ use crate::task_mgr::{self, COMPUTE_REQUEST_RUNTIME};
 use crate::tenant::mgr::ShardSelector;
 use crate::tenant::mgr::TenantManager;
 use crate::tenant::mgr::{GetActiveTenantError, GetTenantError, ShardResolveResult};
-use crate::tenant::timeline::{self, WaitLsnError};
+use crate::tenant::timeline::{self, PageTraceEvent, WaitLsnError};
 use crate::tenant::GetTimelineError;
 use crate::tenant::PageReconstructError;
 use crate::tenant::Timeline;
@@ -618,9 +618,6 @@ impl BatchedFeMessage {
        };
        let throttled = tokio::select! {
            throttled = shard.pagestream_throttle.throttle(tokens) => { throttled }
-            _ = shard.cancel.cancelled() => {
-                return Err(QueryError::Shutdown);
-            }
            _ = cancel.cancelled() => {
                return Err(QueryError::Shutdown);
            }
@@ -1705,6 +1702,18 @@ impl PageServerHandler {
            .query_metrics
            .observe_getpage_batch_start(requests.len());

+        if let Some(page_trace) = timeline.page_trace.load().as_ref() {
+            let time = SystemTime::now();
+            for BatchedGetPageRequest { req, timer: _ } in &requests {
+                let key = rel_block_to_key(req.rel, req.blkno).to_compact();
+                page_trace.send(PageTraceEvent {
+                    key,
+                    effective_lsn,
+                    time,
+                })
+            }
+        }
+
        let results = timeline
            .get_rel_page_at_lsn_batched(
                requests.iter().map(|p| (&p.req.rel, &p.req.blkno)),
--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -26,8 +26,8 @@ use once_cell::sync::Lazy;
 use pageserver_api::{
    config::tenant_conf_defaults::DEFAULT_COMPACTION_THRESHOLD,
    key::{
-        KEY_SIZE, METADATA_KEY_BEGIN_PREFIX, METADATA_KEY_END_PREFIX, NON_INHERITED_RANGE,
-        NON_INHERITED_SPARSE_RANGE,
+        CompactKey, KEY_SIZE, METADATA_KEY_BEGIN_PREFIX, METADATA_KEY_END_PREFIX,
+        NON_INHERITED_RANGE, NON_INHERITED_SPARSE_RANGE,
    },
    keyspace::{KeySpaceAccum, KeySpaceRandomAccum, SparseKeyPartitioning},
    models::{
@@ -434,6 +434,49 @@ pub struct Timeline {

    /// Cf. [`crate::tenant::CreateTimelineIdempotency`].
    pub(crate) create_idempotency: crate::tenant::CreateTimelineIdempotency,
+
+    pub(crate) page_trace: ArcSwap<Option<PageTrace>>,
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub(crate) struct PageTraceEvent {
+    pub(crate) key: CompactKey,
+    pub(crate) effective_lsn: Lsn,
+    pub(crate) time: SystemTime,
+}
+
+/// When one of these is instantiated for a tenant, it will be used to record fine-grained
+/// history of getpage@lsn requests.
+pub(crate) struct PageTrace {
+    size_limit: u64,
+    size: AtomicU64,
+    trace_tx: tokio::sync::mpsc::UnboundedSender<PageTraceEvent>,
+}
+
+impl PageTrace {
+    pub(crate) fn new(
+        size_limit: u64,
+    ) -> (Self, tokio::sync::mpsc::UnboundedReceiver<PageTraceEvent>) {
+        let (trace_tx, trace_rx) = tokio::sync::mpsc::unbounded_channel();
+        let page_trace = Self {
+            size_limit,
+            size: AtomicU64::new(0),
+            trace_tx,
+        };
+
+        (page_trace, trace_rx)
+    }
+
+    pub(crate) fn send(&self, event: PageTraceEvent) {
+        if self.size.load(std::sync::atomic::Ordering::Relaxed) < self.size_limit {
+            self.size.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+
+            if let Err(_e) = self.trace_tx.send(event) {
+                // Ignore errors: if the receiver is gone, we'll just write up to our size limit
+                // and then stop.
+            }
+        }
+    }
 }

 pub type TimelineDeleteProgress = Arc<tokio::sync::Mutex<DeleteTimelineFlow>>;
@@ -2380,6 +2423,8 @@ impl Timeline {
                attach_wal_lag_cooldown,

                create_idempotency,
+
+                page_trace: Default::default(),
            };

            result.repartition_threshold =
--- a/poetry.lock
+++ b/poetry.lock
@@ -2028,13 +2028,13 @@ openapi-schema-validator = ">=0.4.2,<0.5.0"

 [[package]]
 name = "packaging"
-version = "24.2"
+version = "23.0"
 description = "Core utilities for Python packages"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.7"
 files = [
-    {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
-    {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
+    {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"},
+    {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"},
 ]

 [[package]]
--- a/proxy/src/auth/backend/console_redirect.rs
+++ b/proxy/src/auth/backend/console_redirect.rs
@@ -1,8 +1,7 @@
-use std::fmt;
-
 use async_trait::async_trait;
 use postgres_client::config::SslMode;
 use pq_proto::BeMessage as Be;
+use std::fmt;
 use thiserror::Error;
 use tokio::io::{AsyncRead, AsyncWrite};
 use tracing::{info, info_span};
@@ -13,13 +12,10 @@ use crate::auth::IpPattern;
 use crate::cache::Cached;
 use crate::config::AuthenticationConfig;
 use crate::context::RequestContext;
-use crate::control_plane::client::cplane_proxy_v1;
-use crate::control_plane::{self, CachedNodeInfo, NodeInfo};
+use crate::control_plane::{self, client::cplane_proxy_v1, CachedNodeInfo, NodeInfo};
 use crate::error::{ReportableError, UserFacingError};
 use crate::proxy::connect_compute::ComputeConnectBackend;
-use crate::proxy::NeonOptions;
 use crate::stream::PqStream;
-use crate::types::RoleName;
 use crate::{auth, compute, waiters};

 #[derive(Debug, Error)]
@@ -109,16 +105,10 @@ impl ConsoleRedirectBackend {
        ctx: &RequestContext,
        auth_config: &'static AuthenticationConfig,
        client: &mut PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-    ) -> auth::Result<(
-        ConsoleRedirectNodeInfo,
-        ComputeUserInfo,
-        Option<Vec<IpPattern>>,
-    )> {
+    ) -> auth::Result<(ConsoleRedirectNodeInfo, Option<Vec<IpPattern>>)> {
        authenticate(ctx, auth_config, &self.console_uri, client)
            .await
-            .map(|(node_info, user_info, ip_allowlist)| {
-                (ConsoleRedirectNodeInfo(node_info), user_info, ip_allowlist)
-            })
+            .map(|(node_info, ip_allowlist)| (ConsoleRedirectNodeInfo(node_info), ip_allowlist))
    }
 }

@@ -143,7 +133,7 @@ async fn authenticate(
    auth_config: &'static AuthenticationConfig,
    link_uri: &reqwest::Url,
    client: &mut PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-) -> auth::Result<(NodeInfo, ComputeUserInfo, Option<Vec<IpPattern>>)> {
+) -> auth::Result<(NodeInfo, Option<Vec<IpPattern>>)> {
    ctx.set_auth_method(crate::context::AuthMethod::ConsoleRedirect);

    // registering waiter can fail if we get unlucky with rng.
@@ -198,15 +188,8 @@ async fn authenticate(
    let mut config = compute::ConnCfg::new(db_info.host.to_string(), db_info.port);
    config.dbname(&db_info.dbname).user(&db_info.user);

-    let user: RoleName = db_info.user.into();
-    let user_info = ComputeUserInfo {
-        endpoint: db_info.aux.endpoint_id.as_str().into(),
-        user: user.clone(),
-        options: NeonOptions::default(),
-    };
-
    ctx.set_dbname(db_info.dbname.into());
-    ctx.set_user(user);
+    ctx.set_user(db_info.user.into());
    ctx.set_project(db_info.aux.clone());
    info!("woken up a compute node");

@@ -229,7 +212,6 @@ async fn authenticate(
            config,
            aux: db_info.aux,
        },
-        user_info,
        db_info.allowed_ips,
    ))
 }
--- a/proxy/src/compute.rs
+++ b/proxy/src/compute.rs
@@ -24,8 +24,10 @@ use crate::control_plane::messages::MetricsAuxInfo;
 use crate::error::{ReportableError, UserFacingError};
 use crate::metrics::{Metrics, NumDbConnectionsGuard};
 use crate::proxy::neon_option;
+use crate::proxy::NeonOptions;
 use crate::tls::postgres_rustls::MakeRustlsConnect;
 use crate::types::Host;
+use crate::types::{EndpointId, RoleName};

 pub const COULD_NOT_CONNECT: &str = "Couldn't connect to compute node";

@@ -251,7 +253,6 @@ impl ConnCfg {
        ctx: &RequestContext,
        aux: MetricsAuxInfo,
        config: &ComputeConfig,
-        user_info: ComputeUserInfo,
    ) -> Result<PostgresConnection, ConnectionError> {
        let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Compute);
        let (socket_addr, stream, host) = self.connect_raw(config.timeout).await?;
@@ -286,6 +287,28 @@ impl ConnCfg {
            self.0.get_ssl_mode()
        );

+        let compute_info = match parameters.get("user") {
+            Some(user) => {
+                match parameters.get("database") {
+                    Some(database) => {
+                        ComputeUserInfo {
+                            user: RoleName::from(user),
+                            options: NeonOptions::default(), // just a shim, we don't need options
+                            endpoint: EndpointId::from(database),
+                        }
+                    }
+                    None => {
+                        warn!("compute node didn't return database name");
+                        ComputeUserInfo::default()
+                    }
+                }
+            }
+            None => {
+                warn!("compute node didn't return user name");
+                ComputeUserInfo::default()
+            }
+        };
+
        // NB: CancelToken is supposed to hold socket_addr, but we use connect_raw.
        // Yet another reason to rework the connection establishing code.
        let cancel_closure = CancelClosure::new(
@@ -298,7 +321,7 @@ impl ConnCfg {
            },
            vec![], // TODO: deprecated, will be removed
            host.to_string(),
-            user_info,
+            compute_info,
        );

        let connection = PostgresConnection {
--- a/proxy/src/console_redirect_proxy.rs
+++ b/proxy/src/console_redirect_proxy.rs
@@ -195,7 +195,7 @@ pub(crate) async fn handle_client<S: AsyncRead + AsyncWrite + Unpin>(

    ctx.set_db_options(params.clone());

-    let (node_info, user_info, ip_allowlist) = match backend
+    let (user_info, ip_allowlist) = match backend
        .authenticate(ctx, &config.authentication_config, &mut stream)
        .await
    {
@@ -208,12 +208,11 @@ pub(crate) async fn handle_client<S: AsyncRead + AsyncWrite + Unpin>(
    let mut node = connect_to_compute(
        ctx,
        &TcpMechanism {
-            user_info,
            params_compat: true,
            params: &params,
            locks: &config.connect_compute_locks,
        },
-        &node_info,
+        &user_info,
        config.wake_compute_retry_config,
        &config.connect_to_compute,
    )
--- a/proxy/src/control_plane/mod.rs
+++ b/proxy/src/control_plane/mod.rs
@@ -74,11 +74,8 @@ impl NodeInfo {
        &self,
        ctx: &RequestContext,
        config: &ComputeConfig,
-        user_info: ComputeUserInfo,
    ) -> Result<compute::PostgresConnection, compute::ConnectionError> {
-        self.config
-            .connect(ctx, self.aux.clone(), config, user_info)
-            .await
+        self.config.connect(ctx, self.aux.clone(), config).await
    }

    pub(crate) fn reuse_settings(&mut self, other: Self) {
--- a/proxy/src/proxy/connect_compute.rs
+++ b/proxy/src/proxy/connect_compute.rs
@@ -4,7 +4,7 @@ use tokio::time;
 use tracing::{debug, info, warn};

 use super::retry::ShouldRetryWakeCompute;
-use crate::auth::backend::{ComputeCredentialKeys, ComputeUserInfo};
+use crate::auth::backend::ComputeCredentialKeys;
 use crate::compute::{self, PostgresConnection, COULD_NOT_CONNECT};
 use crate::config::{ComputeConfig, RetryConfig};
 use crate::context::RequestContext;
@@ -71,8 +71,6 @@ pub(crate) struct TcpMechanism<'a> {

    /// connect_to_compute concurrency lock
    pub(crate) locks: &'static ApiLocks<Host>,
-
-    pub(crate) user_info: ComputeUserInfo,
 }

 #[async_trait]
@@ -90,7 +88,7 @@ impl ConnectMechanism for TcpMechanism<'_> {
    ) -> Result<PostgresConnection, Self::Error> {
        let host = node_info.config.get_host();
        let permit = self.locks.get_permit(&host).await?;
-        permit.release_result(node_info.connect(ctx, config, self.user_info.clone()).await)
+        permit.release_result(node_info.connect(ctx, config).await)
    }

    fn update_connect_config(&self, config: &mut compute::ConnCfg) {
--- a/proxy/src/proxy/mod.rs
+++ b/proxy/src/proxy/mod.rs
@@ -332,19 +332,16 @@ pub(crate) async fn handle_client<S: AsyncRead + AsyncWrite + Unpin>(
        }
    };

-    let compute_user_info = match &user_info {
-        auth::Backend::ControlPlane(_, info) => &info.info,
-        auth::Backend::Local(_) => unreachable!("local proxy does not run tcp proxy service"),
+    let params_compat = match &user_info {
+        auth::Backend::ControlPlane(_, info) => {
+            info.info.options.get(NeonOptions::PARAMS_COMPAT).is_some()
+        }
+        auth::Backend::Local(_) => false,
    };
-    let params_compat = compute_user_info
-        .options
-        .get(NeonOptions::PARAMS_COMPAT)
-        .is_some();

    let mut node = connect_to_compute(
        ctx,
        &TcpMechanism {
-            user_info: compute_user_info.clone(),
            params_compat,
            params: &params,
            locks: &config.connect_compute_locks,
--- a/proxy/src/redis/notifications.rs
+++ b/proxy/src/redis/notifications.rs
@@ -74,11 +74,7 @@ pub(crate) enum Notification {
    #[serde(rename = "/cancel_session")]
    Cancel(CancelSession),

-    #[serde(
-        other,
-        deserialize_with = "deserialize_unknown_topic",
-        skip_serializing
-    )]
+    #[serde(other, skip_serializing)]
    UnknownTopic,
 }

@@ -127,15 +123,6 @@ where
    serde_json::from_str(&s).map_err(<D::Error as serde::de::Error>::custom)
 }

-// https://github.com/serde-rs/serde/issues/1714
-fn deserialize_unknown_topic<'de, D>(deserializer: D) -> Result<(), D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    deserializer.deserialize_any(serde::de::IgnoredAny)?;
-    Ok(())
-}
-
 struct MessageHandler<C: ProjectInfoCache + Send + Sync + 'static> {
    cache: Arc<C>,
    cancellation_handler: Arc<CancellationHandler<()>>,
@@ -471,30 +458,4 @@ mod tests {

        Ok(())
    }
-
-    #[test]
-    fn parse_unknown_topic() -> anyhow::Result<()> {
-        let with_data = json!({
-            "type": "message",
-            "topic": "/doesnotexist",
-            "data": {
-                "payload": "ignored"
-            },
-            "extra_fields": "something"
-        })
-        .to_string();
-        let result: Notification = serde_json::from_str(&with_data)?;
-        assert_eq!(result, Notification::UnknownTopic);
-
-        let without_data = json!({
-            "type": "message",
-            "topic": "/doesnotexist",
-            "extra_fields": "something"
-        })
-        .to_string();
-        let result: Notification = serde_json::from_str(&without_data)?;
-        assert_eq!(result, Notification::UnknownTopic);
-
-        Ok(())
-    }
 }
--- a/safekeeper/src/bin/safekeeper.rs
+++ b/safekeeper/src/bin/safekeeper.rs
@@ -51,10 +51,12 @@ use utils::{
 #[global_allocator]
 static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

-/// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20).
-#[allow(non_upper_case_globals)]
-#[export_name = "malloc_conf"]
-pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0";
+// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20).
+// TODO: disabled because concurrent CPU profiles cause seg faults. See:
+// https://github.com/neondatabase/neon/issues/10225.
+//#[allow(non_upper_case_globals)]
+//#[export_name = "malloc_conf"]
+//pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0";

 const PID_FILE_NAME: &str = "safekeeper.pid";
 const ID_FILE_NAME: &str = "safekeeper.id";
--- a/storage_controller/client/src/control_api.rs
+++ b/storage_controller/client/src/control_api.rs
@@ -1,6 +1,7 @@
 use pageserver_client::mgmt_api::{self, ResponseErrorMessageExt};
 use reqwest::{Method, Url};
 use serde::{de::DeserializeOwned, Serialize};
+use std::str::FromStr;

 pub struct Client {
    base_url: Url,
@@ -30,11 +31,16 @@ impl Client {
        RQ: Serialize + Sized,
        RS: DeserializeOwned + Sized,
    {
-        let request_path = self
-            .base_url
-            .join(&path)
-            .expect("Failed to build request path");
-        let mut builder = self.client.request(method, request_path);
+        // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out
+        // for general purpose API access.
+        let url = Url::from_str(&format!(
+            "http://{}:{}/{path}",
+            self.base_url.host_str().unwrap(),
+            self.base_url.port().unwrap()
+        ))
+        .unwrap();
+
+        let mut builder = self.client.request(method, url);
        if let Some(body) = body {
            builder = builder.json(&body)
        }
--- a/test_runner/regress/test_compatibility.py
+++ b/test_runner/regress/test_compatibility.py
@@ -141,18 +141,11 @@ def test_create_snapshot(
    neon_env_builder.num_safekeepers = 3
    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)

-    env = neon_env_builder.init_start(
-        initial_tenant_conf={
-            # Miniature layers to enable generating non-trivial layer map without writing lots of data
-            "checkpoint_distance": f"{128 * 1024}",
-            "compaction_threshold": "1",
-            "compaction_target_size": f"{128 * 1024}",
-        }
-    )
+    env = neon_env_builder.init_start()
    endpoint = env.endpoints.create_start("main")

-    pg_bin.run_capture(["pgbench", "--initialize", "--scale=1", endpoint.connstr()])
-    pg_bin.run_capture(["pgbench", "--time=30", "--progress=2", endpoint.connstr()])
+    pg_bin.run_capture(["pgbench", "--initialize", "--scale=10", endpoint.connstr()])
+    pg_bin.run_capture(["pgbench", "--time=60", "--progress=2", endpoint.connstr()])
    pg_bin.run_capture(
        ["pg_dumpall", f"--dbname={endpoint.connstr()}", f"--file={test_output_dir / 'dump.sql'}"]
    )
@@ -164,9 +157,7 @@ def test_create_snapshot(
    pageserver_http = env.pageserver.http_client()

    flush_ep_to_pageserver(env, endpoint, tenant_id, timeline_id)
-    pageserver_http.timeline_checkpoint(
-        tenant_id, timeline_id, wait_until_uploaded=True, force_image_layer_creation=True
-    )
+    pageserver_http.timeline_checkpoint(tenant_id, timeline_id, wait_until_uploaded=True)

    env.endpoints.stop_all()
    for sk in env.safekeepers:
--- a/test_runner/regress/test_lfc_resize.py
+++ b/test_runner/regress/test_lfc_resize.py
@@ -30,7 +30,7 @@ def test_lfc_resize(neon_simple_env: NeonEnv, pg_bin: PgBin):
        ],
    )
    n_resize = 10
-    scale = 20
+    scale = 100

    def run_pgbench(connstr: str):
        log.info(f"Start a pgbench workload on pg {connstr}")
@@ -46,36 +46,17 @@ def test_lfc_resize(neon_simple_env: NeonEnv, pg_bin: PgBin):
    conn = endpoint.connect()
    cur = conn.cursor()

-    def get_lfc_size() -> tuple[int, int]:
-        lfc_file_path = endpoint.lfc_path()
-        lfc_file_size = lfc_file_path.stat().st_size
-        res = subprocess.run(
-            ["ls", "-sk", lfc_file_path], check=True, text=True, capture_output=True
-        )
-        lfc_file_blocks = re.findall("([0-9A-F]+)", res.stdout)[0]
-        log.info(f"Size of LFC file {lfc_file_size}, blocks {lfc_file_blocks}")
-
-        return (lfc_file_size, lfc_file_blocks)
-
    # For as long as pgbench is running, twiddle the LFC size once a second.
    # Note that we launch this immediately, already while the "pgbench -i"
    # initialization step is still running. That's quite a different workload
    # than the actual pgbench benchamark run, so this gives us coverage of both.
    while thread.is_alive():
-        # Vary the LFC size randomly within a range above what we will later
-        # decrease it to.  This should ensure that the final size decrease
-        # is really doing something.
-        size = random.randint(192, 512)
+        size = random.randint(1, 512)
        cur.execute(f"alter system set neon.file_cache_size_limit='{size}MB'")
        cur.execute("select pg_reload_conf()")
        time.sleep(1)
-
    thread.join()

-    # Before shrinking the cache, check that it really is large now
-    (lfc_file_size, lfc_file_blocks) = get_lfc_size()
-    assert int(lfc_file_blocks) > 128 * 1024
-
    # At the end, set it at 100 MB, and perform a final check that the disk usage
    # of the file is in that ballbark.
    #
@@ -85,7 +66,13 @@ def test_lfc_resize(neon_simple_env: NeonEnv, pg_bin: PgBin):
    cur.execute("select pg_reload_conf()")
    nretries = 10
    while True:
-        (lfc_file_size, lfc_file_blocks) = get_lfc_size()
+        lfc_file_path = endpoint.lfc_path()
+        lfc_file_size = lfc_file_path.stat().st_size
+        res = subprocess.run(
+            ["ls", "-sk", lfc_file_path], check=True, text=True, capture_output=True
+        )
+        lfc_file_blocks = re.findall("([0-9A-F]+)", res.stdout)[0]
+        log.info(f"Size of LFC file {lfc_file_size}, blocks {lfc_file_blocks}")
        assert lfc_file_size <= 512 * 1024 * 1024

        if int(lfc_file_blocks) <= 128 * 1024 or nretries == 0:
--- a/test_runner/regress/test_local_file_cache.py
+++ b/test_runner/regress/test_local_file_cache.py
@@ -29,8 +29,8 @@ def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder):
    cur = endpoint.connect().cursor()

    stop = threading.Event()
-    n_rows = 10000
-    n_threads = 5
+    n_rows = 100000
+    n_threads = 20
    n_updates_per_connection = 1000

    cur.execute("CREATE TABLE lfctest (id int4 PRIMARY KEY, n int) WITH (fillfactor=10)")
--- a/test_runner/performance/test_parallel_copy.py
+++ b/test_runner/performance/test_parallel_copy.py
Author	SHA1	Message	Date
John Spray	7dc86ec883	pageserver: API for invoking page trace	2025-01-14 18:04:59 +00:00
John Spray	44f1c4917d	pageserver: add PageTrace machinery	2025-01-14 18:04:59 +00:00
Vlad Lazar	4c093c6314	Merge pull request #10338 from neondatabase/rc/release/2025-01-10 Storage release 2025-01-10	2025-01-10 19:21:43 +00:00
github-actions[bot]	32f58f8228	Storage release 2025-01-10	2025-01-10 06:02:00 +00:00
Erik Grinaker	96c36c0894	Merge pull request #10263 from neondatabase/rc/release/2025-01-03 Storage release 2025-01-03	2025-01-03 20:32:37 +01:00
Erik Grinaker	d719709316	Revert "pageserver: revert flush backpressure (#8550 ) (#10135 )" (#10270 ) This reverts commit `f3ecd5d76a`. It is [suspected](https://neondb.slack.com/archives/C033RQ5SPDH/p1735907405716759) to have caused significant read amplification in the [ingest benchmark](https://neonprod.grafana.net/d/de3mupf4g68e8e/perf-test3a-ingest-benchmark?orgId=1&from=now-30d&to=now&timezone=utc&var-new_project_endpoint_id=ep-solitary-sun-w22bmut6&var-large_tenant_endpoint_id=ep-holy-bread-w203krzs) (specifically during index creation). We will revisit an intermediate improvement here to unblock [upload parallelism](https://github.com/neondatabase/neon/issues/10096) before properly addressing [compaction backpressure](https://github.com/neondatabase/neon/issues/8390).	2025-01-03 16:51:16 +01:00
Erik Grinaker	97912f19fc	pageserver,safekeeper: disable heap profiling (#10268 ) ## Problem Since enabling continuous profiling in staging, we've seen frequent seg faults. This is suspected to be because jemalloc and pprof-rs take a stack trace at the same time, and the handlers aren't signal safe. jemalloc does this probabilistically on every allocation, regardless of whether someone is taking a heap profile, which means that any CPU profile has a chance to cause a seg fault. Touches #10225. ## Summary of changes For now, just disable heap profiles -- CPU profiles are more important, and we need to be able to take them without risking a crash.	2025-01-03 16:51:16 +01:00
github-actions[bot]	49724aa3b6	Storage release 2025-01-03	2025-01-03 06:02:03 +00:00