mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-15 09:22:55 +00:00
306c4f9967
## Problem The scrubber didn't know how to find the latest index_part when generations were in use. ## Summary of changes - Teach the scrubber to do the same dance that pageserver does when finding the latest index_part.json - Teach the scrubber how to understand layer files with generation suffixes. - General improvement to testability: scan_metadata has a machine readable output that the testing `S3Scrubber` wrapper can read. - Existing test coverage of scrubber was false-passing because it just didn't see any data due to prefixing of data in the bucket. Fix that. This is incremental improvement: the more confidence we can have in the scrubber, the more we can use it in integration tests to validate the state of remote storage. --------- Co-authored-by: Arpad Müller <arpad-m@users.noreply.github.com>
43 lines
1.2 KiB
TOML
43 lines
1.2 KiB
TOML
[package]
|
|
name = "s3_scrubber"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[dependencies]
|
|
aws-sdk-s3.workspace = true
|
|
aws-smithy-http.workspace = true
|
|
aws-types.workspace = true
|
|
either.workspace = true
|
|
tokio-rustls.workspace = true
|
|
anyhow.workspace = true
|
|
hex.workspace = true
|
|
thiserror.workspace = true
|
|
rand.workspace = true
|
|
bytes.workspace = true
|
|
bincode.workspace = true
|
|
crc32c.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
workspace_hack.workspace = true
|
|
utils.workspace = true
|
|
async-stream.workspace = true
|
|
tokio-stream.workspace = true
|
|
futures-util.workspace = true
|
|
itertools.workspace = true
|
|
|
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
chrono = { workspace = true, default-features = false, features = ["clock", "serde"] }
|
|
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "json"] }
|
|
aws-config = { workspace = true, default-features = false, features = ["rustls", "credentials-sso"] }
|
|
|
|
pageserver = { path = "../pageserver" }
|
|
remote_storage = { path = "../libs/remote_storage" }
|
|
|
|
tracing.workspace = true
|
|
tracing-subscriber.workspace = true
|
|
clap.workspace = true
|
|
tracing-appender = "0.2"
|
|
histogram = "0.7"
|