mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-06 21:12:55 +00:00
44781518d0
## Problem After a shard split, the pageserver leaves the ancestor shard's content in place. It may be referenced by child shards, but eventually child shards will de-reference most ancestor layers as they write their own data and do GC. We would like to eventually clean up those ancestor layers to reclaim space. ## Summary of changes - Extend the physical GC command with `--mode=full`, which includes cleaning up unreferenced ancestor shard layers - Add test `test_scrubber_physical_gc_ancestors` - Remove colored log output: in testing this is irritating ANSI code spam in logs, and in interactive use doesn't add much. - Refactor storage controller API client code out of storcon_client into a `storage_controller/client` crate - During physical GC of ancestors, call into the storage controller to check that the latest shards seen in S3 reflect the latest state of the tenant, and there is no shard split in progress.
313 lines
10 KiB
TOML
313 lines
10 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"control_plane/storcon_cli",
|
|
"pageserver",
|
|
"pageserver/compaction",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/pagebench",
|
|
"proxy",
|
|
"safekeeper",
|
|
"storage_broker",
|
|
"storage_controller",
|
|
"storage_controller/client",
|
|
"storage_scrubber",
|
|
"workspace_hack",
|
|
"libs/compute_api",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/safekeeper_api",
|
|
"libs/desim",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2021"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
ahash = "0.8"
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.6"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
atomic-take = "1.1.0"
|
|
azure_core = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_identity = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage_blobs = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
flate2 = "1.0.26"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.3", default-features = false, features=["rustls"] }
|
|
aws-sdk-s3 = "1.26"
|
|
aws-sdk-iam = "1.15.0"
|
|
aws-smithy-async = { version = "1.2.1", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.1.9"
|
|
aws-credential-types = "1.2.0"
|
|
aws-sigv4 = { version = "1.2.1", features = ["sign-http"] }
|
|
aws-types = "1.2.0"
|
|
axum = { version = "0.6.20", features = ["ws"] }
|
|
base64 = "0.13.0"
|
|
bincode = "1.3"
|
|
bindgen = "0.65"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.0"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive"] }
|
|
comfy-table = "6.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
crossbeam-deque = "0.8.5"
|
|
crossbeam-utils = "0.8.5"
|
|
dashmap = { version = "5.5.0", features = ["raw-api"] }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fallible-iterator = "0.2"
|
|
framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
hashbrown = "0.14"
|
|
hashlink = "0.9.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.3.1"
|
|
http = {version = "1.1.0", features = ["std"]}
|
|
http-types = { version = "2", default-features = false }
|
|
humantime = "2.1"
|
|
humantime-serde = "1.1.1"
|
|
hyper = "0.14"
|
|
tokio-tungstenite = "0.20.0"
|
|
indexmap = "2"
|
|
inotify = "0.10.2"
|
|
ipnet = "2.9.0"
|
|
itertools = "0.10"
|
|
jsonwebtoken = "9"
|
|
lasso = "0.7"
|
|
leaky-bucket = "1.0.1"
|
|
libc = "0.2"
|
|
md5 = "0.7.0"
|
|
measured = { version = "0.0.22", features=["lasso"] }
|
|
measured-process = { version = "0.0.22" }
|
|
memoffset = "0.8"
|
|
nix = { version = "0.27", features = ["fs", "process", "socket", "signal", "poll"] }
|
|
notify = "6.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.15"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.20.0"
|
|
opentelemetry-otlp = { version = "0.13.0", default-features=false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.12.0"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "51.0.0", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "51.0.0"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pin-project-lite = "0.2"
|
|
procfs = "0.14"
|
|
prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.11"
|
|
rand = "0.8"
|
|
redis = { version = "0.25.2", features = ["tokio-rustls-comp", "keep-alive"] }
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_20"] }
|
|
reqwest-middleware = "0.3.0"
|
|
reqwest-retry = "0.5"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = "0.22"
|
|
rustls-pemfile = "2"
|
|
rustls-split = "0.3"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
sentry = { version = "0.32", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = "2.0"
|
|
serde_assert = "0.5.0"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
strum = "0.24"
|
|
strum_macros = "0.24"
|
|
"subtle" = "2.5.0"
|
|
# Our PR https://github.com/nical/rust_debug/pull/4 has been merged but no new version released yet
|
|
svg_fmt = { git = "https://github.com/nical/rust_debug", rev = "28a7d96eecff2f28e75b1ea09f2d499a60d0e3b4" }
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
task-local-extensions = "0.1.4"
|
|
test-context = "0.3"
|
|
thiserror = "1.0"
|
|
tikv-jemallocator = "0.5"
|
|
tikv-jemalloc-ctl = "0.5"
|
|
tokio = { version = "1.17", features = ["macros"] }
|
|
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.11.0"
|
|
tokio-rustls = "0.25"
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "rt"] }
|
|
toml = "0.7"
|
|
toml_edit = "0.19"
|
|
tonic = {version = "0.9", features = ["tls", "tls-roots"]}
|
|
tower-service = "0.3.2"
|
|
tracing = "0.1"
|
|
tracing-error = "0.2.0"
|
|
tracing-opentelemetry = "0.21.0"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
twox-hash = { version = "1.6.3", default-features = false }
|
|
typed-json = "0.1"
|
|
url = "2.2"
|
|
urlencoding = "2.1"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
rustls-native-certs = "0.7"
|
|
x509-parser = "0.15"
|
|
whoami = "1.5.1"
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.10"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
|
|
## Other git libraries
|
|
heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
desim = { version = "0.1", path = "./libs/desim" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
storage_controller_client = { path = "./storage_controller/client" }
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
criterion = "0.5.1"
|
|
rcgen = "0.12"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.9"
|
|
|
|
[patch.crates-io]
|
|
|
|
# Needed to get `tokio-postgres-rustls` to depend on our fork.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
|
|
# bug fixes for UUID
|
|
parquet = { git = "https://github.com/apache/arrow-rs", branch = "master" }
|
|
parquet_derive = { git = "https://github.com/apache/arrow-rs", branch = "master" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|