jitter

Signed-off-by: Alex Chi Z <chi@neon.tech>

libs/utils/src/backoff.rs

@@ -1,6 +1,7 @@
 use std::fmt::{Debug, Display};
 
 use futures::Future;
+use rand::Rng;
 use tokio_util::sync::CancellationToken;
 
 pub const DEFAULT_BASE_BACKOFF_SECONDS: f64 = 0.1;
@@ -9,7 +10,6 @@ pub const DEFAULT_MAX_BACKOFF_SECONDS: f64 = 3.0;
 pub const DEFAULT_NETWORK_BASE_BACKOFF_SECONDS: f64 = 1.5;
 pub const DEFAULT_NETWORK_MAX_BACKOFF_SECONDS: f64 = 60.0;
 
-
 pub async fn exponential_backoff(
     n: u32,
     base_increment: f64,
@@ -36,8 +36,11 @@ pub async fn exponential_backoff(
 pub fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds: f64) -> f64 {
     if n == 0 {
         0.0
+    } else if base_increment == 0.0 {
+        1.0
     } else {
         (1.0 + base_increment).powf(f64::from(n)).min(max_seconds)
+            + rand::thread_rng().gen_range(0.0..base_increment)
     }
 }
 
@@ -63,7 +66,8 @@ where
         cancel,
         DEFAULT_BASE_BACKOFF_SECONDS,
         DEFAULT_MAX_BACKOFF_SECONDS,
-    ).await
+    )
+    .await
 }
 
 /// Retries passed operation until one of the following conditions are met:
@@ -133,13 +137,7 @@ where
             }
         }
         // sleep and retry
-        exponential_backoff(
-            attempts,
-            base_increment,
-            max_seconds,
-            cancel,
-        )
-        .await;
+        exponential_backoff(attempts, base_increment, max_seconds, cancel).await;
         attempts += 1;
     }
 }

pageserver/src/tenant/storage_layer/layer.rs

@@ -3,12 +3,12 @@ use camino::{Utf8Path, Utf8PathBuf};
 use pageserver_api::keyspace::KeySpace;
 use pageserver_api::models::HistoricLayerInfo;
 use pageserver_api::shard::{ShardIdentity, ShardIndex, TenantShardId};
-use utils::backoff;
 use std::ops::Range;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::{Arc, Weak};
 use std::time::{Duration, SystemTime};
 use tracing::Instrument;
+use utils::backoff;
 use utils::id::TimelineId;
 use utils::lsn::Lsn;
 use utils::sync::{gate, heavier_once_cell};