mirror of
https://github.com/neondatabase/neon.git
synced 2026-05-30 03:20:36 +00:00
743933176e
## Problem - Scrubber's `tidy` command requires presence of a control plane - Scrubber has no tests at all ## Summary of changes - Add re-usable async streams for reading metadata from a bucket - Add a `scan-metadata` command that reads from those streams and calls existing `checks.rs` code to validate metadata, then returns a summary struct for the bucket. Command returns nonzero status if errors are found. - Add an `enable_scrub_on_exit()` function to NeonEnvBuilder so that tests using remote storage can request to have the scrubber run after they finish - Enable remote storarge and scrub_on_exit in test_pageserver_restart and test_pageserver_chaos This is a "toe in the water" of the overall space of validating the scrubber. Later, we should: - Enable scrubbing at end of tests using remote storage by default - Make the success condition stricter than "no errors": tests should declare what tenants+timelines they expect to see in the bucket (or sniff these from the functions tests use to create them) and we should require that the scrubber reports on these particular tenants/timelines. The `tidy` command is untouched in this PR, but it should be refactored later to use similar async streaming interface instead of the current batch-reading approach (the streams are faster with large buckets), and to also be covered by some tests. --------- Co-authored-by: Joonas Koivunen <joonas@neon.tech> Co-authored-by: Alexander Bayandin <alexander@neon.tech> Co-authored-by: Christian Schwarz <christian@neon.tech> Co-authored-by: Conrad Ludgate <conrad@neon.tech>
41 lines
1.2 KiB
TOML
41 lines
1.2 KiB
TOML
[package]
|
|
name = "s3_scrubber"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[dependencies]
|
|
aws-sdk-s3.workspace = true
|
|
aws-smithy-http.workspace = true
|
|
aws-types.workspace = true
|
|
either.workspace = true
|
|
tokio-rustls.workspace = true
|
|
anyhow.workspace = true
|
|
hex.workspace = true
|
|
thiserror.workspace = true
|
|
rand.workspace = true
|
|
bytes.workspace = true
|
|
bincode.workspace = true
|
|
crc32c.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
workspace_hack.workspace = true
|
|
utils.workspace = true
|
|
async-stream.workspace = true
|
|
tokio-stream.workspace = true
|
|
futures-util.workspace = true
|
|
itertools.workspace = true
|
|
|
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
chrono = { workspace = true, default-features = false, features = ["clock", "serde"] }
|
|
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "json"] }
|
|
aws-config = { workspace = true, default-features = false, features = ["rustls", "credentials-sso"] }
|
|
|
|
pageserver = {path="../pageserver"}
|
|
|
|
tracing.workspace = true
|
|
tracing-subscriber.workspace = true
|
|
clap.workspace = true
|
|
tracing-appender = "0.2"
|
|
histogram = "0.7" |