mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 05:52:55 +00:00
44781518d0
## Problem After a shard split, the pageserver leaves the ancestor shard's content in place. It may be referenced by child shards, but eventually child shards will de-reference most ancestor layers as they write their own data and do GC. We would like to eventually clean up those ancestor layers to reclaim space. ## Summary of changes - Extend the physical GC command with `--mode=full`, which includes cleaning up unreferenced ancestor shard layers - Add test `test_scrubber_physical_gc_ancestors` - Remove colored log output: in testing this is irritating ANSI code spam in logs, and in interactive use doesn't add much. - Refactor storage controller API client code out of storcon_client into a `storage_controller/client` crate - During physical GC of ancestors, call into the storage controller to check that the latest shards seen in S3 reflect the latest state of the tenant, and there is no shard split in progress.
55 lines
1.6 KiB
TOML
55 lines
1.6 KiB
TOML
[package]
|
|
name = "storage_scrubber"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[dependencies]
|
|
aws-sdk-s3.workspace = true
|
|
aws-smithy-async.workspace = true
|
|
either.workspace = true
|
|
tokio-rustls.workspace = true
|
|
anyhow.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
thiserror.workspace = true
|
|
rand.workspace = true
|
|
bytes.workspace = true
|
|
bincode.workspace = true
|
|
crc32c.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
workspace_hack.workspace = true
|
|
utils.workspace = true
|
|
async-stream.workspace = true
|
|
tokio-postgres-rustls.workspace = true
|
|
postgres_ffi.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-util = { workspace = true }
|
|
futures-util.workspace = true
|
|
itertools.workspace = true
|
|
camino.workspace = true
|
|
rustls.workspace = true
|
|
rustls-native-certs.workspace = true
|
|
once_cell.workspace = true
|
|
storage_controller_client.workspace = true
|
|
|
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
chrono = { workspace = true, default-features = false, features = ["clock", "serde"] }
|
|
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "json"] }
|
|
aws-config = { workspace = true, default-features = false, features = ["rustls", "sso"] }
|
|
|
|
pageserver = { path = "../pageserver" }
|
|
pageserver_api = { path = "../libs/pageserver_api" }
|
|
remote_storage = { path = "../libs/remote_storage" }
|
|
|
|
tracing.workspace = true
|
|
tracing-subscriber.workspace = true
|
|
clap.workspace = true
|
|
tracing-appender = "0.2"
|
|
histogram = "0.7"
|
|
|
|
futures.workspace = true
|