storcon: Skip the queue for request-initiated reconciles

Merge pull request #10712 from neondatabase/rc/release/2025-02-07
Storage release 2025-02-07
2026-07-07 22:20:36 +00:00 · 2025-02-13 21:56:28 +01:00 · 2025-02-07 18:21:13 +00:00 · 2025-02-07 18:18:45 +00:00 · 2025-02-07 06:02:18 +00:00 · 2025-02-03 16:43:04 +01:00
3 changed files with 91 additions and 58 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -300,9 +300,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

 [[package]]
 name = "aws-config"
-version = "1.5.15"
+version = "1.5.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc47e70fc35d054c8fcd296d47a61711f043ac80534a10b4f741904f81e73a90"
+checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
@@ -311,7 +311,7 @@ dependencies = [
 "aws-sdk-sts",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.60.7",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -342,9 +342,9 @@ dependencies = [

 [[package]]
 name = "aws-runtime"
-version = "1.5.4"
+version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac"
+checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea"
 dependencies = [
 "aws-credential-types",
 "aws-sigv4",
@@ -368,15 +368,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-iam"
-version = "1.60.0"
+version = "1.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a43daa438f8e7e4ebbbcb5c712b3b85db50d62e637a7da4ba9da51095d327460"
+checksum = "fb8a6fea8d335cde419176b1f2c6d2d6e97997719e7df4b51e59064310f48e4a"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
 "aws-smithy-query",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
@@ -391,15 +391,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-kms"
-version = "1.58.0"
+version = "1.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40b7a24700ac548025a47a5c579886f5198895bb1eccd8964dfd71cd66c16912"
+checksum = "3c30f6fd5646b99d9b45ec3a0c22e67112c175b2383100c960d7ee39d96c8d96"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -413,9 +413,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-s3"
-version = "1.68.0"
+version = "1.65.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc5ddf1dc70287dc9a2f953766a1fe15e3e74aef02fd1335f2afa475c9b4f4fc"
+checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
@@ -424,7 +424,7 @@ dependencies = [
 "aws-smithy-checksums",
 "aws-smithy-eventstream",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -447,15 +447,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sso"
-version = "1.57.0"
+version = "1.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c54bab121fe1881a74c338c5f723d1592bf3b53167f80268a1274f404e1acc38"
+checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -469,15 +469,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.58.0"
+version = "1.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c8234fd024f7ac61c4e44ea008029bde934250f371efe7d4a39708397b1080c"
+checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -491,15 +491,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sts"
-version = "1.58.0"
+version = "1.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba60e1d519d6f23a9df712c04fdeadd7872ac911c84b2f62a8bda92e129b7962"
+checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
 "aws-smithy-query",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
@@ -514,9 +514,9 @@ dependencies = [

 [[package]]
 name = "aws-sigv4"
-version = "1.2.8"
+version = "1.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bc5bbd1e4a2648fd8c5982af03935972c24a2f9846b396de661d351ee3ce837"
+checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-eventstream",
@@ -543,9 +543,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-async"
-version = "1.2.4"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e"
+checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c"
 dependencies = [
 "futures-util",
 "pin-project-lite",
@@ -575,9 +575,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-eventstream"
-version = "0.60.6"
+version = "0.60.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a"
+checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90"
 dependencies = [
 "aws-smithy-types",
 "bytes",
@@ -586,9 +586,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-http"
-version = "0.60.12"
+version = "0.60.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc"
+checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6"
 dependencies = [
 "aws-smithy-eventstream",
 "aws-smithy-runtime-api",
@@ -607,9 +607,18 @@ dependencies = [

 [[package]]
 name = "aws-smithy-json"
-version = "0.61.2"
+version = "0.60.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422"
+checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6"
+dependencies = [
+ "aws-smithy-types",
+]
+
+[[package]]
+name = "aws-smithy-json"
+version = "0.61.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095"
 dependencies = [
 "aws-smithy-types",
 ]
@@ -626,9 +635,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-runtime"
-version = "1.7.7"
+version = "1.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e"
+checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45"
 dependencies = [
 "aws-smithy-async",
 "aws-smithy-http",
@@ -670,9 +679,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-types"
-version = "1.2.13"
+version = "1.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042"
+checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510"
 dependencies = [
 "base64-simd",
 "bytes",
@@ -705,9 +714,9 @@ dependencies = [

 [[package]]
 name = "aws-types"
-version = "1.3.5"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f"
+checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-async",
--- a/storage_controller/src/service.rs
+++ b/storage_controller/src/service.rs
@@ -413,6 +413,7 @@ pub struct Service {

    // Limit how many Reconcilers we will spawn concurrently
    reconciler_concurrency: Arc<tokio::sync::Semaphore>,
+    prio_reconciler_concurrency: Arc<tokio::sync::Semaphore>,

    /// Queue of tenants who are waiting for concurrency limits to permit them to reconcile
    /// Send into this queue to promptly attempt to reconcile this shard next time units are available.
@@ -1189,7 +1190,7 @@ impl Service {
                let (nodes, tenants, _scheduler) = locked.parts_mut();
                if let Some(shard) = tenants.get_mut(&tenant_shard_id) {
                    shard.delayed_reconcile = false;
-                    self.maybe_reconcile_shard(shard, nodes);
+                    self.maybe_reconcile_shard(shard, nodes, false);
                }

                if self.reconciler_concurrency.available_permits() == 0 {
@@ -1466,6 +1467,7 @@ impl Service {
            reconciler_concurrency: Arc::new(tokio::sync::Semaphore::new(
                config.reconciler_concurrency,
            )),
+            prio_reconciler_concurrency: Arc::new(tokio::sync::Semaphore::new(1024)),
            delayed_reconcile_tx,
            abort_tx,
            startup_complete: startup_complete.clone(),
@@ -2244,7 +2246,7 @@ impl Service {
            tenants
                .range_mut(TenantShardId::tenant_range(tenant_id))
                .filter_map(|(_shard_id, shard)| {
-                    self.maybe_configured_reconcile_shard(shard, nodes, config)
+                    self.maybe_configured_reconcile_shard(shard, nodes, config, true)
                })
                .collect::<Vec<_>>()
        };
@@ -2713,7 +2715,7 @@ impl Service {

                        shard.schedule(scheduler, &mut schedule_context)?;

-                        let maybe_waiter = self.maybe_reconcile_shard(shard, nodes);
+                        let maybe_waiter = self.maybe_reconcile_shard(shard, nodes, true);
                        if let Some(waiter) = maybe_waiter {
                            waiters.push(waiter);
                        }
@@ -2834,7 +2836,7 @@ impl Service {
            let (nodes, tenants, _scheduler) = locked.parts_mut();
            for (_shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
                shard.config = config.clone();
-                if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes) {
+                if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes, true) {
                    waiters.push(waiter);
                }
            }
@@ -3116,7 +3118,7 @@ impl Service {
                debug_assert!(shard.intent.get_attached().is_none());
                debug_assert!(shard.intent.get_secondary().is_empty());

-                if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes) {
+                if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes, true) {
                    detach_waiters.push(waiter);
                }
            }
@@ -3268,7 +3270,7 @@ impl Service {

            // In case scheduling is being switched back on, try it now.
            shard.schedule(scheduler, &mut schedule_context).ok();
-            self.maybe_reconcile_shard(shard, nodes);
+            self.maybe_reconcile_shard(shard, nodes, true);
        }

        Ok(())
@@ -4317,7 +4319,7 @@ impl Service {
                    tracing::warn!("Failed to schedule {tenant_shard_id} during shard abort: {e}")
                }

-                self.maybe_reconcile_shard(shard, nodes);
+                self.maybe_reconcile_shard(shard, nodes, true);
            }

            // We don't expect any new_shard_count shards to exist here, but drop them just in case
@@ -4483,7 +4485,8 @@ impl Service {
                        tracing::warn!("Failed to schedule child shard {child}: {e}");
                    }
                    // In the background, attach secondary locations for the new shards
-                    if let Some(waiter) = self.maybe_reconcile_shard(&mut child_state, nodes) {
+                    if let Some(waiter) = self.maybe_reconcile_shard(&mut child_state, nodes, true)
+                    {
                        waiters.push(waiter);
                    }

@@ -4848,7 +4851,7 @@ impl Service {
                shard.intent.clear_secondary(scheduler);

                // Run Reconciler to execute detach fo secondary locations.
-                if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes) {
+                if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes, true) {
                    waiters.push(waiter);
                }
            }
@@ -5114,7 +5117,7 @@ impl Service {
                shard.sequence = shard.sequence.next();
            }

-            self.maybe_reconcile_shard(shard, nodes)
+            self.maybe_reconcile_shard(shard, nodes, true)
        };

        if let Some(waiter) = waiter {
@@ -5177,7 +5180,7 @@ impl Service {
                );
            }

-            self.maybe_reconcile_shard(shard, nodes)
+            self.maybe_reconcile_shard(shard, nodes, true)
        };

        if let Some(waiter) = waiter {
@@ -5589,7 +5592,7 @@ impl Service {
                            )
                        }

-                        self.maybe_reconcile_shard(shard, nodes);
+                        self.maybe_reconcile_shard(shard, nodes, true);
                    }

                    // Here we remove an existing observed location for the node we're removing, and it will
@@ -5958,7 +5961,10 @@ impl Service {
                                    tracing::warn!(%tenant_shard_id, "Scheduling error when marking pageserver {} offline: {e}", node_id);
                                }
                                Ok(()) => {
-                                    if self.maybe_reconcile_shard(tenant_shard, nodes).is_some() {
+                                    if self
+                                        .maybe_reconcile_shard(tenant_shard, nodes, false)
+                                        .is_some()
+                                    {
                                        tenants_affected += 1;
                                    };
                                }
@@ -5989,7 +5995,7 @@ impl Service {

                    if let Some(observed_loc) = tenant_shard.observed.locations.get_mut(&node_id) {
                        if observed_loc.conf.is_none() {
-                            self.maybe_reconcile_shard(tenant_shard, nodes);
+                            self.maybe_reconcile_shard(tenant_shard, nodes, false);
                        }
                    }
                }
@@ -6353,8 +6359,14 @@ impl Service {
        &self,
        shard: &mut TenantShard,
        nodes: &Arc<HashMap<NodeId, Node>>,
+        high_priority: bool,
    ) -> Option<ReconcilerWaiter> {
-        self.maybe_configured_reconcile_shard(shard, nodes, ReconcilerConfig::default())
+        self.maybe_configured_reconcile_shard(
+            shard,
+            nodes,
+            ReconcilerConfig::default(),
+            high_priority,
+        )
    }

    /// Wrap [`TenantShard`] reconciliation methods with acquisition of [`Gate`] and [`ReconcileUnits`],
@@ -6363,6 +6375,7 @@ impl Service {
        shard: &mut TenantShard,
        nodes: &Arc<HashMap<NodeId, Node>>,
        reconciler_config: ReconcilerConfig,
+        high_priority: bool,
    ) -> Option<ReconcilerWaiter> {
        let reconcile_needed = shard.get_reconcile_needed(nodes);

@@ -6374,7 +6387,13 @@ impl Service {
            }
        };

-        let units = match self.reconciler_concurrency.clone().try_acquire_owned() {
+        let acq = if high_priority {
+            self.prio_reconciler_concurrency.clone().try_acquire_owned()
+        } else {
+            self.reconciler_concurrency.clone().try_acquire_owned()
+        };
+
+        let units = match acq {
            Ok(u) => ReconcileUnits::new(u),
            Err(_) => {
                tracing::info!(tenant_id=%shard.tenant_shard_id.tenant_id, shard_id=%shard.tenant_shard_id.shard_slug(),
@@ -6468,7 +6487,10 @@ impl Service {

            // Eventual consistency: if an earlier reconcile job failed, and the shard is still
            // dirty, spawn another rone
-            if self.maybe_reconcile_shard(shard, &pageservers).is_some() {
+            if self
+                .maybe_reconcile_shard(shard, &pageservers, false)
+                .is_some()
+            {
                reconciles_spawned += 1;
            } else if shard.delayed_reconcile {
                // Shard wanted to reconcile but for some reason couldn't.
@@ -6554,7 +6576,7 @@ impl Service {
            tracing::info!(tenant_shard_id=%tenant_shard_id, "Applying optimization: {optimization:?}");
            if shard.apply_optimization(scheduler, optimization) {
                optimizations_applied += 1;
-                if self.maybe_reconcile_shard(shard, nodes).is_some() {
+                if self.maybe_reconcile_shard(shard, nodes, false).is_some() {
                    reconciles_spawned += 1;
                }
            }
@@ -7221,6 +7243,7 @@ impl Service {
                            tenant_shard,
                            nodes,
                            reconciler_config,
+                            false,
                        );
                        if let Some(some) = waiter {
                            waiters.push(some);
@@ -7513,6 +7536,7 @@ impl Service {
                                        tenant_shard,
                                        nodes,
                                        reconciler_config,
+                                        false,
                                    ) {
                                        waiters.push(waiter);
                                    }
--- a/storage_controller/src/service/chaos_injector.rs
+++ b/storage_controller/src/service/chaos_injector.rs
@@ -88,7 +88,7 @@ impl ChaosInjector {

        shard.intent.demote_attached(scheduler, old_location);
        shard.intent.promote_attached(scheduler, new_location);
-        self.service.maybe_reconcile_shard(shard, nodes);
+        self.service.maybe_reconcile_shard(shard, nodes, false);
    }

    async fn inject_chaos(&mut self) {
Author	SHA1	Message	Date
John Spray	6b5118313e	storcon: Skip the queue for request-initiated reconciles	2025-02-13 21:56:28 +01:00
John Spray	a54853abd5	Merge pull request #10712 from neondatabase/rc/release/2025-02-07 Storage release 2025-02-07	2025-02-07 18:21:13 +00:00
Arpad Müller	69007f7ac8	Revert recent AWS SDK update (#10724 ) We've been seeing some regressions in staging since the AWS SDK updates: https://github.com/neondatabase/neon/issues/10695 . We aren't sure the regression was caused by the SDK update, but the issues do involve S3, so it's not unlikely. By reverting the SDK update we find out whether it was really the SDK update, or something else. Reverts the two PRs: * https://github.com/neondatabase/neon/pull/10588 * https://github.com/neondatabase/neon/pull/10699 https://neondb.slack.com/archives/C08C2G15M6U/p1738576986047179	2025-02-07 18:18:45 +00:00
github-actions[bot]	d255fa4b7e	Storage release 2025-02-07	2025-02-07 06:02:18 +00:00
Arpad Müller	40d6b3a34e	Merge pull request #10602 from neondatabase/rc/release/2025-01-31 Storage release 2025-01-31	2025-02-03 16:43:04 +01:00
github-actions[bot]	a018878e27	Storage release 2025-01-31	2025-01-31 06:02:08 +00:00
Christian Schwarz	e5b3eb1e64	Merge pull request #10500 from neondatabase/rc/release/2025-01-24 Storage release 2025-01-24	2025-01-25 00:54:56 +01:00
github-actions[bot]	f35e1356a1	Storage release 2025-01-24	2025-01-24 06:02:13 +00:00
Christian Schwarz	4dec0dddc6	Merge pull request #10447 from neondatabase/releases/2025-01-20-hotfix Release: storage hotfix 2025-01-20	2025-01-20 15:55:44 +01:00
Christian Schwarz	e0c504af38	fix(page_service / handle): panic when parallel client disconnect & Timeline shutdown Refs - fixes https://github.com/neondatabase/neon/issues/10444	2025-01-20 14:37:16 +01:00
Alex Chi Z.	3399eea2ed	Merge pull request #10436 from neondatabase/rc/release/2025-01-17 Storage release 2025-01-17	2025-01-17 12:36:17 -05:00
Alex Chi Z	6a29c809d5	Merge branch 'release' of https://github.com/neondatabase/neon into rc/release/2025-01-17	2025-01-17 10:44:25 -05:00
github-actions[bot]	a62c01df4c	Storage release 2025-01-17	2025-01-17 06:02:11 +00:00
Vlad Lazar	4c093c6314	Merge pull request #10338 from neondatabase/rc/release/2025-01-10 Storage release 2025-01-10	2025-01-10 19:21:43 +00:00
github-actions[bot]	32f58f8228	Storage release 2025-01-10	2025-01-10 06:02:00 +00:00
Erik Grinaker	96c36c0894	Merge pull request #10263 from neondatabase/rc/release/2025-01-03 Storage release 2025-01-03	2025-01-03 20:32:37 +01:00
Erik Grinaker	d719709316	Revert "pageserver: revert flush backpressure (#8550 ) (#10135 )" (#10270 ) This reverts commit `f3ecd5d76a`. It is [suspected](https://neondb.slack.com/archives/C033RQ5SPDH/p1735907405716759) to have caused significant read amplification in the [ingest benchmark](https://neonprod.grafana.net/d/de3mupf4g68e8e/perf-test3a-ingest-benchmark?orgId=1&from=now-30d&to=now&timezone=utc&var-new_project_endpoint_id=ep-solitary-sun-w22bmut6&var-large_tenant_endpoint_id=ep-holy-bread-w203krzs) (specifically during index creation). We will revisit an intermediate improvement here to unblock [upload parallelism](https://github.com/neondatabase/neon/issues/10096) before properly addressing [compaction backpressure](https://github.com/neondatabase/neon/issues/8390).	2025-01-03 16:51:16 +01:00
Erik Grinaker	97912f19fc	pageserver,safekeeper: disable heap profiling (#10268 ) ## Problem Since enabling continuous profiling in staging, we've seen frequent seg faults. This is suspected to be because jemalloc and pprof-rs take a stack trace at the same time, and the handlers aren't signal safe. jemalloc does this probabilistically on every allocation, regardless of whether someone is taking a heap profile, which means that any CPU profile has a chance to cause a seg fault. Touches #10225. ## Summary of changes For now, just disable heap profiles -- CPU profiles are more important, and we need to be able to take them without risking a crash.	2025-01-03 16:51:16 +01:00
github-actions[bot]	49724aa3b6	Storage release 2025-01-03	2025-01-03 06:02:03 +00:00