Use jemalloc-pprof for symbolization

storcon: infrastructure for safekeeper specific JWT tokens (#10905 )
Safekeepers only respond to requests with the per-token scope, or the `safekeeperdata` JWT scope. Therefore, add infrastructure in the storage controller for safekeeper JWTs. Also, rename the ambiguous `jwt_token` to `pageserver_jwt_token`. Part of #9011 Related: https://github.com/neondatabase/cloud/issues/24727
2026-07-11 08:00:38 +00:00 · 2025-02-21 17:13:56 +01:00 · 2025-02-21 11:02:02 +00:00 · 2025-02-21 11:01:57 +00:00 · 2025-02-21 09:03:54 +00:00
9 changed files with 99 additions and 65 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3263,8 +3263,7 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 [[package]]
 name = "jemalloc_pprof"
 version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a883828bd6a4b957cd9f618886ff19e5f3ebd34e06ba0e855849e049fef32fb"
+source = "git+https://github.com/erikgrinaker/rust-jemalloc-pprof?branch=symbolize#0b146a1e2013bbc7fc8dc45f208f868c0b8ed193"
 dependencies = [
 "anyhow",
 "libc",
@@ -3453,8 +3452,7 @@ dependencies = [
 [[package]]
 name = "mappings"
 version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce9229c438fbf1c333926e2053c4c091feabbd40a1b590ec62710fea2384af9e"
+source = "git+https://github.com/erikgrinaker/rust-jemalloc-pprof?branch=symbolize#0b146a1e2013bbc7fc8dc45f208f868c0b8ed193"
 dependencies = [
 "anyhow",
 "libc",
@@ -4729,10 +4727,10 @@ dependencies = [
 [[package]]
 name = "pprof_util"
 version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65c568b3f8c1c37886ae07459b1946249e725c315306b03be5632f84c239f781"
+source = "git+https://github.com/erikgrinaker/rust-jemalloc-pprof?branch=symbolize#0b146a1e2013bbc7fc8dc45f208f868c0b8ed193"
 dependencies = [
 "anyhow",
+ "backtrace",
 "flate2",
 "num",
 "paste",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -116,7 +116,7 @@ inferno = "0.12.0"
 ipnet = "2.10.0"
 itertools = "0.10"
 itoa = "1.0.11"
-jemalloc_pprof = "0.6"
+jemalloc_pprof = { git = "https://github.com/erikgrinaker/rust-jemalloc-pprof", branch = "symbolize", version = "0.6", features = ["symbolize"] }
 jsonwebtoken = "9"
 lasso = "0.7"
 libc = "0.2"
--- a/compute/compute-node.Dockerfile
+++ b/compute/compute-node.Dockerfile
@@ -395,15 +395,22 @@ RUN case "${PG_VERSION:?}" in \
    cd plv8-src && \
    if [[ "${PG_VERSION:?}" < "v17" ]]; then patch -p1 < /ext-src/plv8-3.1.10.patch; fi

-FROM pg-build AS plv8-build
+# Step 1: Build the vendored V8 engine. It doesn't depend on PostgreSQL, so use
+# 'build-deps' as the base. This enables caching and avoids unnecessary rebuilds.
+# (The V8 engine takes a very long time to build)
+FROM build-deps AS plv8-build
 ARG PG_VERSION
+WORKDIR /ext-src/plv8-src
 RUN apt update && \
    apt install --no-install-recommends --no-install-suggests -y \
    ninja-build python3-dev libncurses5 binutils clang \
    && apt clean && rm -rf /var/lib/apt/lists/*
-
 COPY --from=plv8-src /ext-src/ /ext-src/
-WORKDIR /ext-src/plv8-src
+RUN make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) v8
+
+# Step 2: Build the PostgreSQL-dependent parts
+COPY --from=pg-build /usr/local/pgsql /usr/local/pgsql
+ENV PATH="/usr/local/pgsql/bin:$PATH"
 RUN \
    # generate and copy upgrade scripts
    make generate_upgrades && \
--- a/libs/http-utils/src/endpoint.rs
+++ b/libs/http-utils/src/endpoint.rs
@@ -495,19 +495,10 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
        }

        Format::Pprof => {
-            let data = tokio::task::spawn_blocking(move || {
-                let bytes = prof_ctl.dump_pprof()?;
-                // Symbolize the profile.
-                // TODO: consider moving this upstream to jemalloc_pprof and avoiding the
-                // serialization roundtrip.
-                let profile = pprof::decode(&bytes)?;
-                let profile = pprof::symbolize(profile)?;
-                let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
-                pprof::encode(&profile)
-            })
-            .await
-            .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
-            .map_err(ApiError::InternalServerError)?;
+            let data = tokio::task::spawn_blocking(move || prof_ctl.dump_pprof())
+                .await
+                .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
+                .map_err(ApiError::InternalServerError)?;
            Response::builder()
                .status(200)
                .header(CONTENT_TYPE, "application/octet-stream")
@@ -520,8 +511,6 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
            let body = tokio::task::spawn_blocking(move || {
                let bytes = prof_ctl.dump_pprof()?;
                let profile = pprof::decode(&bytes)?;
-                let profile = pprof::symbolize(profile)?;
-                let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
                let mut opts = inferno::flamegraph::Options::default();
                opts.title = "Heap inuse".to_string();
                opts.count_name = "bytes".to_string();
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -672,6 +672,10 @@ fn start_pageserver(
            }
        });

+        // Allocate a bunch of memory.
+        let alloc = allocate(256 * 1024 * 1024);
+        println!("allocated {}b", alloc.len());
+
        // Wait for cancellation signal and shut down the pageserver.
        //
        // This cancels the `shutdown_pageserver` cancellation tree. Right now that tree doesn't
@@ -695,6 +699,17 @@ fn start_pageserver(
    })
 }

+#[inline(never)]
+fn allocate(size: usize) -> Vec<u8> {
+    allocate_inline(size)
+}
+
+#[inline(always)]
+fn allocate_inline(size: usize) -> Vec<u8> {
+    println!("allocating {size}b");
+    vec![9; size]
+}
+
 async fn create_remote_storage_client(
    conf: &'static PageServerConf,
 ) -> anyhow::Result<GenericRemoteStorage> {
--- a/storage_controller/src/http.rs
+++ b/storage_controller/src/http.rs
@@ -598,7 +598,10 @@ async fn handle_tenant_timeline_passthrough(

    let _timer = latency.start_timer(labels.clone());

-    let client = mgmt_api::Client::new(node.base_url(), service.get_config().jwt_token.as_deref());
+    let client = mgmt_api::Client::new(
+        node.base_url(),
+        service.get_config().pageserver_jwt_token.as_deref(),
+    );
    let resp = client.get_raw(path).await.map_err(|e|
        // We return 503 here because if we can't successfully send a request to the pageserver,
        // either we aren't available or the pageserver is unavailable.
@@ -1354,10 +1357,7 @@ async fn handle_safekeeper_scheduling_policy(
        .set_safekeeper_scheduling_policy(id, body.scheduling_policy)
        .await?;

-    Ok(Response::builder()
-        .status(StatusCode::NO_CONTENT)
-        .body(Body::empty())
-        .unwrap())
+    json_response(StatusCode::OK, ())
 }

 /// Common wrapper for request handlers that call into Service and will operate on tenants: they must only
--- a/storage_controller/src/main.rs
+++ b/storage_controller/src/main.rs
@@ -53,6 +53,10 @@ struct Cli {
    #[arg(long)]
    jwt_token: Option<String>,

+    /// Token for authenticating this service with the safekeepers it controls
+    #[arg(long)]
+    safekeeper_jwt_token: Option<String>,
+
    /// Token for authenticating this service with the control plane, when calling
    /// the compute notification endpoint
    #[arg(long)]
@@ -153,7 +157,8 @@ impl Default for StrictMode {
 struct Secrets {
    database_url: String,
    public_key: Option<JwtAuth>,
-    jwt_token: Option<String>,
+    pageserver_jwt_token: Option<String>,
+    safekeeper_jwt_token: Option<String>,
    control_plane_jwt_token: Option<String>,
    peer_jwt_token: Option<String>,
 }
@@ -161,6 +166,7 @@ struct Secrets {
 impl Secrets {
    const DATABASE_URL_ENV: &'static str = "DATABASE_URL";
    const PAGESERVER_JWT_TOKEN_ENV: &'static str = "PAGESERVER_JWT_TOKEN";
+    const SAFEKEEPER_JWT_TOKEN_ENV: &'static str = "SAFEKEEPER_JWT_TOKEN";
    const CONTROL_PLANE_JWT_TOKEN_ENV: &'static str = "CONTROL_PLANE_JWT_TOKEN";
    const PEER_JWT_TOKEN_ENV: &'static str = "PEER_JWT_TOKEN";
    const PUBLIC_KEY_ENV: &'static str = "PUBLIC_KEY";
@@ -184,7 +190,14 @@ impl Secrets {
        let this = Self {
            database_url,
            public_key,
-            jwt_token: Self::load_secret(&args.jwt_token, Self::PAGESERVER_JWT_TOKEN_ENV),
+            pageserver_jwt_token: Self::load_secret(
+                &args.jwt_token,
+                Self::PAGESERVER_JWT_TOKEN_ENV,
+            ),
+            safekeeper_jwt_token: Self::load_secret(
+                &args.safekeeper_jwt_token,
+                Self::SAFEKEEPER_JWT_TOKEN_ENV,
+            ),
            control_plane_jwt_token: Self::load_secret(
                &args.control_plane_jwt_token,
                Self::CONTROL_PLANE_JWT_TOKEN_ENV,
@@ -264,11 +277,17 @@ async fn async_main() -> anyhow::Result<()> {

    let secrets = Secrets::load(&args).await?;

+    // TODO: once we've rolled out the safekeeper JWT token everywhere, put it into the validation code below
+    tracing::info!(
+        "safekeeper_jwt_token set: {:?}",
+        secrets.safekeeper_jwt_token.is_some()
+    );
+
    // Validate required secrets and arguments are provided in strict mode
    match strict_mode {
        StrictMode::Strict
            if (secrets.public_key.is_none()
-                || secrets.jwt_token.is_none()
+                || secrets.pageserver_jwt_token.is_none()
                || secrets.control_plane_jwt_token.is_none()) =>
        {
            // Production systems should always have secrets configured: if public_key was not set
@@ -293,7 +312,8 @@ async fn async_main() -> anyhow::Result<()> {
    }

    let config = Config {
-        jwt_token: secrets.jwt_token,
+        pageserver_jwt_token: secrets.pageserver_jwt_token,
+        safekeeper_jwt_token: secrets.safekeeper_jwt_token,
        control_plane_jwt_token: secrets.control_plane_jwt_token,
        peer_jwt_token: secrets.peer_jwt_token,
        compute_hook_url: args.compute_hook_url,
--- a/storage_controller/src/reconciler.rs
+++ b/storage_controller/src/reconciler.rs
@@ -296,7 +296,7 @@ impl Reconciler {
                        .location_config(tenant_shard_id, config.clone(), flush_ms, lazy)
                        .await
                },
-                &self.service_config.jwt_token,
+                &self.service_config.pageserver_jwt_token,
                1,
                3,
                timeout,
@@ -417,7 +417,7 @@ impl Reconciler {
        let client = PageserverClient::new(
            node.get_id(),
            node.base_url(),
-            self.service_config.jwt_token.as_deref(),
+            self.service_config.pageserver_jwt_token.as_deref(),
        );

        client
@@ -440,7 +440,7 @@ impl Reconciler {
        let client = PageserverClient::new(
            node.get_id(),
            node.base_url(),
-            self.service_config.jwt_token.as_deref(),
+            self.service_config.pageserver_jwt_token.as_deref(),
        );

        let timelines = client.timeline_list(&tenant_shard_id).await?;
@@ -478,7 +478,7 @@ impl Reconciler {
                            )
                            .await
                    },
-                    &self.service_config.jwt_token,
+                    &self.service_config.pageserver_jwt_token,
                    1,
                    3,
                    request_download_timeout * 2,
@@ -771,7 +771,7 @@ impl Reconciler {
            let observed_conf = match attached_node
                .with_client_retries(
                    |client| async move { client.get_location_config(tenant_shard_id).await },
-                    &self.service_config.jwt_token,
+                    &self.service_config.pageserver_jwt_token,
                    1,
                    1,
                    Duration::from_secs(5),
@@ -1099,7 +1099,7 @@ impl Reconciler {
            match origin
                .with_client_retries(
                    |client| async move { client.get_location_config(tenant_shard_id).await },
-                    &self.service_config.jwt_token,
+                    &self.service_config.pageserver_jwt_token,
                    1,
                    3,
                    Duration::from_secs(5),
--- a/storage_controller/src/service.rs
+++ b/storage_controller/src/service.rs
@@ -348,7 +348,12 @@ pub struct Config {
    // All pageservers managed by one instance of this service must have
    // the same public key.  This JWT token will be used to authenticate
    // this service to the pageservers it manages.
-    pub jwt_token: Option<String>,
+    pub pageserver_jwt_token: Option<String>,
+
+    // All safekeepers managed by one instance of this service must have
+    // the same public key. This JWT token will be used to authenticate
+    // this service to the safekeepers it manages.
+    pub safekeeper_jwt_token: Option<String>,

    // This JWT token will be used to authenticate this service to the control plane.
    pub control_plane_jwt_token: Option<String>,
@@ -882,7 +887,7 @@ impl Service {
                    let response = node
                        .with_client_retries(
                            |client| async move { client.list_location_config().await },
-                            &self.config.jwt_token,
+                            &self.config.pageserver_jwt_token,
                            1,
                            5,
                            timeout,
@@ -983,7 +988,7 @@ impl Service {
            let client = PageserverClient::new(
                node.get_id(),
                node.base_url(),
-                self.config.jwt_token.as_deref(),
+                self.config.pageserver_jwt_token.as_deref(),
            );
            match client
                .location_config(
@@ -1553,14 +1558,14 @@ impl Service {
        let reconcilers_cancel = cancel.child_token();

        let heartbeater_ps = Heartbeater::new(
-            config.jwt_token.clone(),
+            config.pageserver_jwt_token.clone(),
            config.max_offline_interval,
            config.max_warming_up_interval,
            cancel.clone(),
        );

        let heartbeater_sk = Heartbeater::new(
-            config.jwt_token.clone(),
+            config.safekeeper_jwt_token.clone(),
            config.max_offline_interval,
            config.max_warming_up_interval,
            cancel.clone(),
@@ -1907,7 +1912,7 @@ impl Service {
        let configs = match node
            .with_client_retries(
                |client| async move { client.list_location_config().await },
-                &self.config.jwt_token,
+                &self.config.pageserver_jwt_token,
                1,
                5,
                SHORT_RECONCILE_TIMEOUT,
@@ -1965,7 +1970,7 @@ impl Service {
                            .location_config(tenant_shard_id, config, None, false)
                            .await
                    },
-                    &self.config.jwt_token,
+                    &self.config.pageserver_jwt_token,
                    1,
                    5,
                    SHORT_RECONCILE_TIMEOUT,
@@ -3100,7 +3105,7 @@ impl Service {
                let client = PageserverClient::new(
                    node.get_id(),
                    node.base_url(),
-                    self.config.jwt_token.as_deref(),
+                    self.config.pageserver_jwt_token.as_deref(),
                );

                tracing::info!("Doing time travel recovery for shard {tenant_shard_id}",);
@@ -3161,7 +3166,7 @@ impl Service {
            let client = PageserverClient::new(
                node.get_id(),
                node.base_url(),
-                self.config.jwt_token.as_deref(),
+                self.config.pageserver_jwt_token.as_deref(),
            );
            futs.push(async move {
                let result = client
@@ -3284,7 +3289,7 @@ impl Service {
                        .tenant_delete(TenantShardId::unsharded(tenant_id))
                        .await
                },
-                &self.config.jwt_token,
+                &self.config.pageserver_jwt_token,
                1,
                3,
                RECONCILE_TIMEOUT,
@@ -3503,7 +3508,7 @@ impl Service {
            let timeline_info = create_one(
                shard_zero_tid,
                shard_zero_locations,
-                self.config.jwt_token.clone(),
+                self.config.pageserver_jwt_token.clone(),
                create_req.clone(),
            )
            .await?;
@@ -3519,7 +3524,7 @@ impl Service {
            // Create timeline on remaining shards with number >0
            if !targets.0.is_empty() {
                // If we had multiple shards, issue requests for the remainder now.
-                let jwt = &self.config.jwt_token;
+                let jwt = &self.config.pageserver_jwt_token;
                self.tenant_for_shards(
                    targets
                        .0
@@ -3602,7 +3607,7 @@ impl Service {
                        tenant_shard_id,
                        timeline_id,
                        node,
-                        self.config.jwt_token.clone(),
+                        self.config.pageserver_jwt_token.clone(),
                        req.clone(),
                    ))
                })
@@ -3683,7 +3688,7 @@ impl Service {
                        tenant_shard_id,
                        timeline_id,
                        node,
-                        self.config.jwt_token.clone(),
+                        self.config.pageserver_jwt_token.clone(),
                    ))
                })
                .await?;
@@ -3757,7 +3762,7 @@ impl Service {
                    tenant_shard_id,
                    timeline_id,
                    node,
-                    self.config.jwt_token.clone(),
+                    self.config.pageserver_jwt_token.clone(),
                    dir,
                ))
            })
@@ -3872,7 +3877,7 @@ impl Service {
            futs.push(async move {
                node.with_client_retries(
                    |client| op(tenant_shard_id, client),
-                    &self.config.jwt_token,
+                    &self.config.pageserver_jwt_token,
                    warn_threshold,
                    max_retries,
                    timeout,
@@ -4121,7 +4126,7 @@ impl Service {
                        tenant_shard_id,
                        timeline_id,
                        node,
-                        self.config.jwt_token.clone(),
+                        self.config.pageserver_jwt_token.clone(),
                    ))
                })
                .await?;
@@ -4143,7 +4148,7 @@ impl Service {
                shard_zero_tid,
                timeline_id,
                shard_zero_locations.latest.node,
-                self.config.jwt_token.clone(),
+                self.config.pageserver_jwt_token.clone(),
            )
            .await?;
            Ok(shard_zero_status)
@@ -4542,7 +4547,7 @@ impl Service {

                        client.location_config(child_id, config, None, false).await
                    },
-                    &self.config.jwt_token,
+                    &self.config.pageserver_jwt_token,
                    1,
                    10,
                    Duration::from_secs(5),
@@ -5142,7 +5147,7 @@ impl Service {
            let client = PageserverClient::new(
                node.get_id(),
                node.base_url(),
-                self.config.jwt_token.as_deref(),
+                self.config.pageserver_jwt_token.as_deref(),
            );
            let response = client
                .tenant_shard_split(
@@ -5468,7 +5473,7 @@ impl Service {
        let client = PageserverClient::new(
            node.get_id(),
            node.base_url(),
-            self.config.jwt_token.as_deref(),
+            self.config.pageserver_jwt_token.as_deref(),
        );

        let scan_result = client
@@ -7094,7 +7099,7 @@ impl Service {
        match attached_node
            .with_client_retries(
                |client| async move { client.tenant_heatmap_upload(tenant_shard_id).await },
-                &self.config.jwt_token,
+                &self.config.pageserver_jwt_token,
                3,
                10,
                SHORT_RECONCILE_TIMEOUT,
@@ -7130,7 +7135,7 @@ impl Service {
                            )
                            .await
                    },
-                    &self.config.jwt_token,
+                    &self.config.pageserver_jwt_token,
                    3,
                    10,
                    SHORT_RECONCILE_TIMEOUT,
@@ -7185,7 +7190,7 @@ impl Service {
                        let request = request_ref.clone();
                        client.top_tenant_shards(request.clone()).await
                    },
-                    &self.config.jwt_token,
+                    &self.config.pageserver_jwt_token,
                    3,
                    3,
                    Duration::from_secs(5),
@@ -7358,7 +7363,7 @@ impl Service {
        match node
            .with_client_retries(
                |client| async move { client.tenant_secondary_status(tenant_shard_id).await },
-                &self.config.jwt_token,
+                &self.config.pageserver_jwt_token,
                1,
                3,
                Duration::from_millis(250),
Author	SHA1	Message	Date
Erik Grinaker	5e5053eaff	Use jemalloc-pprof for symbolization	2025-02-21 17:13:56 +01:00
Arpad Müller	ff3819efc7	storcon: infrastructure for safekeeper specific JWT tokens (#10905 ) Safekeepers only respond to requests with the per-token scope, or the `safekeeperdata` JWT scope. Therefore, add infrastructure in the storage controller for safekeeper JWTs. Also, rename the ambiguous `jwt_token` to `pageserver_jwt_token`. Part of #9011 Related: https://github.com/neondatabase/cloud/issues/24727	2025-02-21 11:02:02 +00:00
Arpad Müller	f927ae6e15	Return a json response in scheduling_policy handler (#10904 ) Return an empty json response in the `scheduling_policy` handler. This prevents errors of the form: ``` Error: receive body: error decoding response body: EOF while parsing a value at line 1 column 0 ``` when setting the scheduling policy via the `storcon_cli`. part of #9011.	2025-02-21 11:01:57 +00:00
Heikki Linnakangas	61d385caea	Split plv8 build into two parts (#10920 ) Plv8 consists of two parts: 1. the V8 engine, which is built from vendored sources, and 2. the PostgreSQL extension. Split those into two separate steps in the Dockerfile. The first step doesn't need any PostgreSQL sources or any other files from the neon repository, just the build tools and the upstream plv8 sources. Use the build-deps image as the base for that step, so that the layer can be cached and doesn't need to be rebuilt every time. This is worthwhile because the V8 build takes a very long time.	2025-02-21 09:03:54 +00:00