rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2025-12-22 21:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

resolve tenant-snapshot conflict with list_timeline_blobs index parsing

Browse Source 
Signed-off-by: Yuchen Liang <yuchen@neon.tech>
This commit is contained in:
Yuchen Liang
2024-08-02 14:24:05 -04:00
parent b2ea4bb366
commit 422a8443dd
4 changed files with 36 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
storage_scrubber/src/checks.rs
View File

@@ -172,8 +172,11 @@ pub(crate) async fn branch_cleanup_and_check_errors(
} }
} }
BlobDataParseResult::Relic => {} BlobDataParseResult::Relic => {}
BlobDataParseResult::Incorrect(parse_errors) => result.errors.extend( BlobDataParseResult::Incorrect {
parse_errors errors,
s3_layers: _,
} => result.errors.extend(
errors
.into_iter() .into_iter()
.map(|error| format!("parse error: {error}")), .map(|error| format!("parse error: {error}")),
), ),
@@ -300,7 +303,10 @@ pub(crate) enum BlobDataParseResult {
}, },
/// The remains of a deleted Timeline (i.e. an initdb archive only) /// The remains of a deleted Timeline (i.e. an initdb archive only)
Relic, Relic,
Incorrect(Vec<String>), Incorrect {
errors: Vec<String>,
s3_layers: HashSet<(LayerName, Generation)>,
},
} }
pub(crate) fn parse_layer_object_name(name: &str) -> Result<(LayerName, Generation), String> { pub(crate) fn parse_layer_object_name(name: &str) -> Result<(LayerName, Generation), String> {
@@ -443,7 +449,7 @@ pub(crate) async fn list_timeline_blobs(
} }
Ok(S3TimelineBlobData { Ok(S3TimelineBlobData {
blob_data: BlobDataParseResult::Incorrect(errors), blob_data: BlobDataParseResult::Incorrect { errors, s3_layers },
unused_index_keys: index_part_keys, unused_index_keys: index_part_keys,
unknown_keys, unknown_keys,
}) })

14
storage_scrubber/src/pageserver_physical_gc.rs
View File

@@ -389,10 +389,13 @@ async fn gc_ancestor(
// Post-deletion tenant location: don't try and GC it. // Post-deletion tenant location: don't try and GC it.
continue; continue;
} }
BlobDataParseResult::Incorrect(reasons) => { BlobDataParseResult::Incorrect {
errors,
s3_layers: _, // TODO(yuchen): could still check references to these s3 layers?
} => {
// Our primary purpose isn't to report on bad data, but log this rather than skipping silently // Our primary purpose isn't to report on bad data, but log this rather than skipping silently
tracing::warn!( tracing::warn!(
"Skipping ancestor GC for timeline {ttid}, bad metadata: {reasons:?}" "Skipping ancestor GC for timeline {ttid}, bad metadata: {errors:?}"
); );
continue; continue;
} }
@@ -518,9 +521,12 @@ pub async fn pageserver_physical_gc(
// Post-deletion tenant location: don't try and GC it. // Post-deletion tenant location: don't try and GC it.
return Ok(summary); return Ok(summary);
} }
BlobDataParseResult::Incorrect(reasons) => { BlobDataParseResult::Incorrect {
errors,
s3_layers: _,
} => {
// Our primary purpose isn't to report on bad data, but log this rather than skipping silently // Our primary purpose isn't to report on bad data, but log this rather than skipping silently
tracing::warn!("Skipping timeline {ttid}, bad metadata: {reasons:?}"); tracing::warn!("Skipping timeline {ttid}, bad metadata: {errors:?}");
return Ok(summary); return Ok(summary);
} }
}; };

22
storage_scrubber/src/scan_pageserver_metadata.rs
View File

@@ -290,13 +290,21 @@ pub async fn scan_metadata(
} }
} }
if let BlobDataParseResult::Parsed { match &data.blob_data {
index_part: _index_part, BlobDataParseResult::Parsed {
index_part_generation: _index_part_generation, index_part: _index_part,
s3_layers, index_part_generation: _index_part_generation,
} = &data.blob_data s3_layers,
{ } => {
tenant_objects.push(ttid, s3_layers.clone()); tenant_objects.push(ttid, s3_layers.clone());
}
BlobDataParseResult::Relic => (),
BlobDataParseResult::Incorrect {
errors: _,
s3_layers,
} => {
tenant_objects.push(ttid, s3_layers.clone());
}
} }
tenant_timeline_results.push((ttid, data)); tenant_timeline_results.push((ttid, data));
} }

2
storage_scrubber/src/tenant_snapshot.rs
View File

@@ -269,7 +269,7 @@ impl SnapshotDownloader {
.context("Downloading timeline")?; .context("Downloading timeline")?;
} }
BlobDataParseResult::Relic => {} BlobDataParseResult::Relic => {}
BlobDataParseResult::Incorrect(_) => { BlobDataParseResult::Incorrect { .. } => {
tracing::error!("Bad metadata in timeline {ttid}"); tracing::error!("Bad metadata in timeline {ttid}");
} }
}; };