From c71f637d65168fe51bce1bec336e279e80ea8de0 Mon Sep 17 00:00:00 2001
From: Arthur Petukhovsky <petuhovskiy@yandex.ru>
Date: Sun, 11 Sep 2022 23:11:59 +0000
Subject: [PATCH] Fix timeline deletion

---
 libs/utils/src/postgres_backend.rs     |  6 ++---
 safekeeper/src/http/routes.rs          | 12 ++++++---
 safekeeper/src/timelines_global_map.rs | 35 ++++++++++++++++++--------
 3 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/libs/utils/src/postgres_backend.rs b/libs/utils/src/postgres_backend.rs
index 322d01e664..083615acf5 100644
--- a/libs/utils/src/postgres_backend.rs
+++ b/libs/utils/src/postgres_backend.rs
@@ -433,9 +433,9 @@ impl PostgresBackend {
                     // We also don't want to log full stacktrace when the error is primitive,
                     // such as usual connection closed.
                     let short_error = format!("{:#}", e);
-                    if e.root_cause()
-                        .to_string()
-                        .contains("connection closed unexpectedly")
+                    let root_cause = e.root_cause().to_string();
+                    if root_cause.contains("connection closed unexpectedly")
+                        || root_cause.contains("Broken pipe (os error 32)")
                     {
                         error!(
                             "query handler for '{}' failed: {}",
diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs
index a7d1d75732..ec19d8dadf 100644
--- a/safekeeper/src/http/routes.rs
+++ b/safekeeper/src/http/routes.rs
@@ -147,7 +147,9 @@ async fn timeline_delete_force_handler(
     );
     check_permission(&request, Some(zttid.tenant_id))?;
     ensure_no_body(&mut request).await?;
-    let resp = GlobalTimelines::delete_force(&zttid).map_err(ApiError::from_err)?;
+    let resp = tokio::task::spawn_blocking(move || GlobalTimelines::delete_force(&zttid))
+        .await
+        .map_err(ApiError::from_err)??;
     json_response(StatusCode::OK, resp)
 }
 
@@ -159,10 +161,14 @@ async fn tenant_delete_force_handler(
     let tenant_id = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
     ensure_no_body(&mut request).await?;
+    let delete_info = tokio::task::spawn_blocking(move || {
+        GlobalTimelines::delete_force_all_for_tenant(&tenant_id)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
     json_response(
         StatusCode::OK,
-        GlobalTimelines::delete_force_all_for_tenant(&tenant_id)
-            .map_err(ApiError::from_err)?
+        delete_info
             .iter()
             .map(|(zttid, resp)| (format!("{}", zttid.timeline_id), *resp))
             .collect::<HashMap<String, TimelineDeleteForceResult>>(),
diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs
index e594abfd2a..2f2b1dcfcb 100644
--- a/safekeeper/src/timelines_global_map.rs
+++ b/safekeeper/src/timelines_global_map.rs
@@ -251,21 +251,34 @@ impl GlobalTimelines {
 
     /// Cancels timeline, then deletes the corresponding data directory.
     pub fn delete_force(zttid: &ZTenantTimelineId) -> Result<TimelineDeleteForceResult> {
-        let timeline = TIMELINES_STATE.lock().unwrap().get(zttid)?;
+        let tli_res = TIMELINES_STATE.lock().unwrap().get(zttid);
+        match tli_res {
+            Ok(timeline) => {
+                // Take a lock and finish the deletion holding this mutex.
+                let mut shared_state = timeline.write_shared_state();
 
-        // Take a lock and finish the deletion holding this mutex.
-        let mut shared_state = timeline.write_shared_state();
+                info!("deleting timeline {}", zttid);
+                let (dir_existed, was_active) = timeline.delete_from_disk(&mut shared_state)?;
 
-        info!("deleting timeline {}", zttid);
-        let (dir_existed, was_active) = timeline.delete_from_disk(&mut shared_state)?;
+                // Remove timeline from the map.
+                TIMELINES_STATE.lock().unwrap().timelines.remove(zttid);
 
-        // Remove timeline from the map.
-        TIMELINES_STATE.lock().unwrap().timelines.remove(zttid);
+                Ok(TimelineDeleteForceResult {
+                    dir_existed,
+                    was_active,
+                })
+            }
+            Err(_) => {
+                // Timeline is not memory, but it may still exist on disk in broken state.
+                let dir_path = TIMELINES_STATE.lock().unwrap().conf.timeline_dir(zttid);
+                let dir_existed = delete_dir(dir_path)?;
 
-        Ok(TimelineDeleteForceResult {
-            dir_existed,
-            was_active,
-        })
+                Ok(TimelineDeleteForceResult {
+                    dir_existed,
+                    was_active: false,
+                })
+            }
+        }
     }
 
     /// Deactivates and deletes all timelines for the tenant. Returns map of all timelines which