Fix timeline deletion

libs/utils/src/postgres_backend.rs

@@ -433,9 +433,9 @@ impl PostgresBackend {
                     // We also don't want to log full stacktrace when the error is primitive,
                     // such as usual connection closed.
                     let short_error = format!("{:#}", e);
-                    if e.root_cause()
-                        .to_string()
-                        .contains("connection closed unexpectedly")
+                    let root_cause = e.root_cause().to_string();
+                    if root_cause.contains("connection closed unexpectedly")
+                        || root_cause.contains("Broken pipe (os error 32)")
                     {
                         error!(
                             "query handler for '{}' failed: {}",

safekeeper/src/http/routes.rs

@@ -147,7 +147,9 @@ async fn timeline_delete_force_handler(
     );
     check_permission(&request, Some(zttid.tenant_id))?;
     ensure_no_body(&mut request).await?;
-    let resp = GlobalTimelines::delete_force(&zttid).map_err(ApiError::from_err)?;
+    let resp = tokio::task::spawn_blocking(move || GlobalTimelines::delete_force(&zttid))
+        .await
+        .map_err(ApiError::from_err)??;
     json_response(StatusCode::OK, resp)
 }
 
@@ -159,10 +161,14 @@ async fn tenant_delete_force_handler(
     let tenant_id = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
     ensure_no_body(&mut request).await?;
+    let delete_info = tokio::task::spawn_blocking(move || {
+        GlobalTimelines::delete_force_all_for_tenant(&tenant_id)
+    })
+    .await
+    .map_err(ApiError::from_err)??;
     json_response(
         StatusCode::OK,
-        GlobalTimelines::delete_force_all_for_tenant(&tenant_id)
-            .map_err(ApiError::from_err)?
+        delete_info
             .iter()
             .map(|(zttid, resp)| (format!("{}", zttid.timeline_id), *resp))
             .collect::<HashMap<String, TimelineDeleteForceResult>>(),

safekeeper/src/timelines_global_map.rs

@@ -251,21 +251,34 @@ impl GlobalTimelines {
 
     /// Cancels timeline, then deletes the corresponding data directory.
     pub fn delete_force(zttid: &ZTenantTimelineId) -> Result<TimelineDeleteForceResult> {
-        let timeline = TIMELINES_STATE.lock().unwrap().get(zttid)?;
+        let tli_res = TIMELINES_STATE.lock().unwrap().get(zttid);
+        match tli_res {
+            Ok(timeline) => {
+                // Take a lock and finish the deletion holding this mutex.
+                let mut shared_state = timeline.write_shared_state();
 
-        // Take a lock and finish the deletion holding this mutex.
-        let mut shared_state = timeline.write_shared_state();
+                info!("deleting timeline {}", zttid);
+                let (dir_existed, was_active) = timeline.delete_from_disk(&mut shared_state)?;
 
-        info!("deleting timeline {}", zttid);
-        let (dir_existed, was_active) = timeline.delete_from_disk(&mut shared_state)?;
+                // Remove timeline from the map.
+                TIMELINES_STATE.lock().unwrap().timelines.remove(zttid);
 
-        // Remove timeline from the map.
-        TIMELINES_STATE.lock().unwrap().timelines.remove(zttid);
+                Ok(TimelineDeleteForceResult {
+                    dir_existed,
+                    was_active,
+                })
+            }
+            Err(_) => {
+                // Timeline is not memory, but it may still exist on disk in broken state.
+                let dir_path = TIMELINES_STATE.lock().unwrap().conf.timeline_dir(zttid);
+                let dir_existed = delete_dir(dir_path)?;
 
-        Ok(TimelineDeleteForceResult {
-            dir_existed,
-            was_active,
-        })
+                Ok(TimelineDeleteForceResult {
+                    dir_existed,
+                    was_active: false,
+                })
+            }
+        }
     }
 
     /// Deactivates and deletes all timelines for the tenant. Returns map of all timelines which