From 9880653e11535886655bc95bd3768182ae5ea853 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Thu, 19 Sep 2024 15:53:51 -0400 Subject: [PATCH] fix(storage-scrubber): handle race of index part download Signed-off-by: Alex Chi Z --- storage_scrubber/src/checks.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/storage_scrubber/src/checks.rs b/storage_scrubber/src/checks.rs index 15dfb101b5..2d5238ef5d 100644 --- a/storage_scrubber/src/checks.rs +++ b/storage_scrubber/src/checks.rs @@ -478,9 +478,21 @@ pub(crate) async fn list_timeline_blobs( if let Some(index_part_object_key) = index_part_object.as_ref() { let index_part_bytes = - download_object_with_retries(remote_client, &index_part_object_key.key) + match download_object_with_retries(remote_client, &index_part_object_key.key) .await - .context("index_part.json download")?; + .context("index_part.json download") + { + Ok(index_part_bytes) => index_part_bytes, + Err(e) => { + // It is possible that the branch gets deleted in-between we list the objects and we download the index part file. + errors.push(format!("failed to download : {e}")); + return Ok(RemoteTimelineBlobData { + blob_data: BlobDataParseResult::Incorrect { errors, s3_layers }, + unused_index_keys: index_part_keys, + unknown_keys, + }); + } + }; match serde_json::from_slice(&index_part_bytes) { Ok(index_part) => {