From c2d7aadd709a08d3c5bba0869bde9c1cf0872f51 Mon Sep 17 00:00:00 2001
From: John Spray <john@neon.tech>
Date: Mon, 17 Jun 2024 09:09:56 +0100
Subject: [PATCH] pageserver: add api-hang failpoint

---
 pageserver/src/http/routes.rs | 10 +++++++++-
 pageserver/src/tenant/mgr.rs  |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs
index 657708c0d6..2c1951e779 100644
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -47,6 +47,7 @@ use tenant_size_model::{SizeResult, StorageModel};
 use tokio_util::sync::CancellationToken;
 use tracing::*;
 use utils::auth::JwtAuth;
+use utils::failpoint_support;
 use utils::failpoint_support::failpoints_handler;
 use utils::http::endpoint::prometheus_metrics_handler;
 use utils::http::endpoint::request_span;
@@ -2587,7 +2588,9 @@ where
     R: std::future::Future<Output = Result<Response<Body>, ApiError>> + Send + 'static,
     H: FnOnce(Request<Body>, CancellationToken) -> R + Send + Sync + 'static,
 {
-    if request.uri() != &"/v1/failpoints".parse::<Uri>().unwrap() {
+    if request.uri() != &"/v1/failpoints".parse::<Uri>().unwrap()
+        && request.uri() != &"/v1/status".parse::<Uri>().unwrap()
+    {
         fail::fail_point!("api-503", |_| Err(ApiError::ResourceUnavailable(
             "failpoint".into()
         )));
@@ -2595,6 +2598,11 @@ where
         fail::fail_point!("api-500", |_| Err(ApiError::InternalServerError(
             anyhow::anyhow!("failpoint")
         )));
+
+        // This failpoint is meant to check that callers handle hanging requests properly, but not
+        // to block pageserver clean shutdown: hence it respects the tenant manager cancellation token.
+        let state = get_state(&request);
+        failpoint_support::sleep_millis_async!("api-hang", &state.tenant_manager.cancel);
     }
 
     // Spawn a new task to handle the request, to protect the handler from unexpected
diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs
index 4520bb9295..650577fc56 100644
--- a/pageserver/src/tenant/mgr.rs
+++ b/pageserver/src/tenant/mgr.rs
@@ -298,7 +298,7 @@ pub struct TenantManager {
     // This is for edge cases like tenant deletion.  In normal cases (within a Tenant lifetime),
     // tenants have their own cancellation tokens, which we fire individually in [`Self::shutdown`], or
     // when the tenant detaches.
-    cancel: CancellationToken,
+    pub(crate) cancel: CancellationToken,
 }
 
 fn emergency_generations(