logging to debug test_pageserver_restarts_under_worload

pageserver/src/lib.rs

@@ -356,6 +356,25 @@ async fn timed<Fut: std::future::Future>(
     }
 }
 
+/// Like [`timed`], but the warning timeout only starts after `cancel` has been cancelled.
+async fn timed_after_cancellation<Fut: std::future::Future>(
+    fut: Fut,
+    name: &str,
+    warn_at: std::time::Duration,
+    cancel: &CancellationToken,
+) -> <Fut as std::future::Future>::Output {
+    let mut fut = std::pin::pin!(fut);
+
+    tokio::select! {
+        _ = cancel.cancelled() => {
+            timed(fut, name, warn_at).await
+        }
+        ret = &mut fut => {
+            ret
+        }
+    }
+}
+
 #[cfg(test)]
 mod timed_tests {
     use super::timed;

pageserver/src/page_service.rs

@@ -44,7 +44,6 @@ use utils::{
 };
 
 use crate::auth::check_permission;
-use crate::basebackup;
 use crate::basebackup::BasebackupError;
 use crate::config::PageServerConf;
 use crate::context::{DownloadBehavior, RequestContext};
@@ -62,6 +61,7 @@ use crate::tenant::timeline::{self, WaitLsnError};
 use crate::tenant::GetTimelineError;
 use crate::tenant::PageReconstructError;
 use crate::tenant::Timeline;
+use crate::{basebackup, timed_after_cancellation};
 use pageserver_api::key::rel_block_to_key;
 use pageserver_api::reltag::{BlockNumber, RelTag, SlruKind};
 use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID;
@@ -1101,6 +1101,8 @@ impl PageServerHandler {
             protocol_pipelining_mode,
         } = pipelining_config;
 
+        let cancel = self.cancel.clone();
+
         let (requests_tx, mut requests_rx) = tokio::sync::mpsc::channel(1);
         let read_messages = {
             let cancel = self.cancel.child_token();
@@ -1244,8 +1246,23 @@ impl PageServerHandler {
         let executor_res;
         match protocol_pipelining_mode {
             PageServiceProtocolPipeliningMode::ConcurrentFutures => {
-                (read_messages_res, _, executor_res) =
-                    tokio::join!(read_messages, batcher, executor);
+                (read_messages_res, _, executor_res) = {
+                    macro_rules! timed {
+                        ($fut:expr, $what:literal) => {
+                            timed_after_cancellation(
+                                $fut,
+                                $what,
+                                Duration::from_millis(100),
+                                &cancel,
+                            )
+                        };
+                    }
+                    tokio::join!(
+                        timed!(read_messages, "read-messages"),
+                        timed!(batcher, "batcher"),
+                        timed!(executor, "executor"),
+                    )
+                }
             }
             PageServiceProtocolPipeliningMode::Tasks => {
                 // cancelled via sensitivity to self.cancel