From 1d68f52b57e91fe19b5a1695d75ab92775be7fd8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 7 Nov 2023 14:25:51 +0000 Subject: [PATCH] pageserver: move deletion failpoint inside backoff (#5814) ## Problem When enabled, this failpoint would busy-spin in a loop that emits log messages. ## Summary of changes Move the failpoint inside a backoff::exponential block: it will still spam the log, but at much lower rate. --------- Co-authored-by: Joonas Koivunen --- pageserver/src/deletion_queue/deleter.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pageserver/src/deletion_queue/deleter.rs b/pageserver/src/deletion_queue/deleter.rs index 81cb016d49..57421b1547 100644 --- a/pageserver/src/deletion_queue/deleter.rs +++ b/pageserver/src/deletion_queue/deleter.rs @@ -55,21 +55,24 @@ impl Deleter { /// Wrap the remote `delete_objects` with a failpoint async fn remote_delete(&self) -> Result<(), anyhow::Error> { - fail::fail_point!("deletion-queue-before-execute", |_| { - info!("Skipping execution, failpoint set"); - metrics::DELETION_QUEUE - .remote_errors - .with_label_values(&["failpoint"]) - .inc(); - Err(anyhow::anyhow!("failpoint hit")) - }); - // A backoff::retry is used here for two reasons: // - To provide a backoff rather than busy-polling the API on errors // - To absorb transient 429/503 conditions without hitting our error // logging path for issues deleting objects. backoff::retry( - || async { self.remote_storage.delete_objects(&self.accumulator).await }, + || async { + fail::fail_point!("deletion-queue-before-execute", |_| { + info!("Skipping execution, failpoint set"); + + metrics::DELETION_QUEUE + .remote_errors + .with_label_values(&["failpoint"]) + .inc(); + Err(anyhow::anyhow!("failpoint: deletion-queue-before-execute")) + }); + + self.remote_storage.delete_objects(&self.accumulator).await + }, |_| false, 3, 10,