safekeeper: exclude API (#10757)

## Problem

https://github.com/neondatabase/neon/pull/10241 added configuration
switch endpoint, but it didn't delete timeline if node was excluded.

## Summary of changes

Add separate /exclude API endpoint which similarly accepts membership
configuration where sk is supposed by be excluded. Implementation
deletes the timeline locally.

Some more small related tweaks:
- make mconf switch API PUT instead of POST as it is idempotent;
- return 409 if switch was refused instead of 200 with requested &
current;
- remove unused was_active flag from delete response;
- remove meaningless _force suffix from delete functions names;
- reuse timeline.rs delete_dir function in timelines_global_map instead
of its own copy.

part of https://github.com/neondatabase/neon/issues/9965

safekeeper/src/http/routes.rs

@@ -19,7 +19,7 @@ use safekeeper_api::models::{
     AcceptorStateStatus, PullTimelineRequest, SafekeeperStatus, SkTimelineInfo, TermSwitchApiEntry,
     TimelineCopyRequest, TimelineCreateRequest, TimelineStatus, TimelineTermBumpRequest,
 };
-use safekeeper_api::{ServerInfo, models};
+use safekeeper_api::{ServerInfo, membership, models};
 use storage_broker::proto::{SafekeeperTimelineInfo, TenantTimelineId as ProtoTenantTimelineId};
 use tokio::sync::mpsc;
 use tokio::task;
@@ -32,7 +32,7 @@ use utils::lsn::Lsn;
 
 use crate::debug_dump::TimelineDigestRequest;
 use crate::safekeeper::TermLsn;
-use crate::timelines_global_map::TimelineDeleteForceResult;
+use crate::timelines_global_map::{DeleteOrExclude, TimelineDeleteResult};
 use crate::{
     GlobalTimelines, SafeKeeperConf, copy_timeline, debug_dump, patch_control_file, pull_timeline,
 };
@@ -73,10 +73,13 @@ async fn tenant_delete_handler(mut request: Request<Body>) -> Result<Response<Bo
     check_permission(&request, Some(tenant_id))?;
     ensure_no_body(&mut request).await?;
     let global_timelines = get_global_timelines(&request);
-    // FIXME: `delete_force_all_for_tenant` can return an error for multiple different reasons;
-    // Using an `InternalServerError` should be fixed when the types support it
+    let action = if only_local {
+        DeleteOrExclude::DeleteLocal
+    } else {
+        DeleteOrExclude::Delete
+    };
     let delete_info = global_timelines
-        .delete_force_all_for_tenant(&tenant_id, only_local)
+        .delete_all_for_tenant(&tenant_id, action)
         .await
         .map_err(ApiError::InternalServerError)?;
     json_response(
@@ -84,7 +87,7 @@ async fn tenant_delete_handler(mut request: Request<Body>) -> Result<Response<Bo
         delete_info
             .iter()
             .map(|(ttid, resp)| (format!("{}", ttid.timeline_id), *resp))
-            .collect::<HashMap<String, TimelineDeleteForceResult>>(),
+            .collect::<HashMap<String, TimelineDeleteResult>>(),
     )
 }
 
@@ -208,12 +211,15 @@ async fn timeline_delete_handler(mut request: Request<Body>) -> Result<Response<
     check_permission(&request, Some(ttid.tenant_id))?;
     ensure_no_body(&mut request).await?;
     let global_timelines = get_global_timelines(&request);
-    // FIXME: `delete_force` can fail from both internal errors and bad requests. Add better
-    // error handling here when we're able to.
+    let action = if only_local {
+        DeleteOrExclude::DeleteLocal
+    } else {
+        DeleteOrExclude::Delete
+    };
     let resp = global_timelines
-        .delete(&ttid, only_local)
+        .delete_or_exclude(&ttid, action)
         .await
-        .map_err(ApiError::InternalServerError)?;
+        .map_err(ApiError::from)?;
     json_response(StatusCode::OK, resp)
 }
 
@@ -267,6 +273,64 @@ async fn timeline_snapshot_handler(request: Request<Body>) -> Result<Response<Bo
     Ok(response)
 }
 
+/// Error type for delete_or_exclude: either generation conflict or something
+/// internal.
+#[derive(thiserror::Error, Debug)]
+pub enum DeleteOrExcludeError {
+    #[error("refused to switch into excluding mconf {requested}, current: {current}")]
+    Conflict {
+        requested: membership::Configuration,
+        current: membership::Configuration,
+    },
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+
+/// Convert DeleteOrExcludeError to ApiError.
+impl From<DeleteOrExcludeError> for ApiError {
+    fn from(de: DeleteOrExcludeError) -> ApiError {
+        match de {
+            DeleteOrExcludeError::Conflict {
+                requested: _,
+                current: _,
+            } => ApiError::Conflict(de.to_string()),
+            DeleteOrExcludeError::Other(e) => ApiError::InternalServerError(e),
+        }
+    }
+}
+
+/// Remove timeline locally after this node has been excluded from the
+/// membership configuration. The body is the same as in the membership endpoint
+/// -- conf where node is excluded -- and in principle single ep could be used
+/// for both actions, but since this is a data deletion op let's keep them
+/// separate.
+async fn timeline_exclude_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let ttid = TenantTimelineId::new(
+        parse_request_param(&request, "tenant_id")?,
+        parse_request_param(&request, "timeline_id")?,
+    );
+    check_permission(&request, Some(ttid.tenant_id))?;
+
+    let global_timelines = get_global_timelines(&request);
+    let data: models::TimelineMembershipSwitchRequest = json_request(&mut request).await?;
+    let my_id = get_conf(&request).my_id;
+    // If request doesn't exclude us, membership switch endpoint should be used
+    // instead.
+    if data.mconf.contains(my_id) {
+        return Err(ApiError::Forbidden(format!(
+            "refused to switch into {}, node {} is member of it",
+            data.mconf, my_id
+        )));
+    }
+    let action = DeleteOrExclude::Exclude(data.mconf);
+
+    let resp = global_timelines
+        .delete_or_exclude(&ttid, action)
+        .await
+        .map_err(ApiError::from)?;
+    json_response(StatusCode::OK, resp)
+}
+
 /// Consider switching timeline membership configuration to the provided one.
 async fn timeline_membership_handler(
     mut request: Request<Body>,
@@ -281,12 +345,29 @@ async fn timeline_membership_handler(
     let tli = global_timelines.get(ttid).map_err(ApiError::from)?;
 
     let data: models::TimelineMembershipSwitchRequest = json_request(&mut request).await?;
+    let my_id = get_conf(&request).my_id;
+    // If request excludes us, exclude endpoint should be used instead.
+    if !data.mconf.contains(my_id) {
+        return Err(ApiError::Forbidden(format!(
+            "refused to switch into {}, node {} is not a member of it",
+            data.mconf, my_id
+        )));
+    }
+    let req_gen = data.mconf.generation;
     let response = tli
         .membership_switch(data.mconf)
         .await
         .map_err(ApiError::InternalServerError)?;
 
-    json_response(StatusCode::OK, response)
+    // Return 409 if request was ignored.
+    if req_gen == response.current_conf.generation {
+        json_response(StatusCode::OK, response)
+    } else {
+        Err(ApiError::Conflict(format!(
+            "request to switch into {} ignored, current generation {}",
+            req_gen, response.current_conf.generation
+        )))
+    }
 }
 
 async fn timeline_copy_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -637,11 +718,14 @@ pub fn make_router(
         .post("/v1/pull_timeline", |r| {
             request_span(r, timeline_pull_handler)
         })
+        .put("/v1/tenant/:tenant_id/timeline/:timeline_id/exclude", |r| {
+            request_span(r, timeline_exclude_handler)
+        })
         .get(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/snapshot/:destination_id",
             |r| request_span(r, timeline_snapshot_handler),
         )
-        .post(
+        .put(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/membership",
             |r| request_span(r, timeline_membership_handler),
         )